001
014
015 package com.liferay.portlet.documentlibrary.store;
016
017 import com.liferay.portal.kernel.exception.PortalException;
018 import com.liferay.portal.kernel.exception.SystemException;
019 import com.liferay.portal.kernel.log.Log;
020 import com.liferay.portal.kernel.log.LogFactoryUtil;
021 import com.liferay.portal.kernel.util.CharPool;
022 import com.liferay.portal.kernel.util.DateUtil;
023 import com.liferay.portal.kernel.util.FileUtil;
024 import com.liferay.portal.kernel.util.GetterUtil;
025 import com.liferay.portal.kernel.util.LocaleUtil;
026 import com.liferay.portal.kernel.util.PropsKeys;
027 import com.liferay.portal.kernel.util.ReleaseInfo;
028 import com.liferay.portal.kernel.util.StreamUtil;
029 import com.liferay.portal.kernel.util.StringBundler;
030 import com.liferay.portal.kernel.util.StringPool;
031 import com.liferay.portal.kernel.util.StringUtil;
032 import com.liferay.portal.kernel.util.SystemProperties;
033 import com.liferay.portal.kernel.util.Time;
034 import com.liferay.portal.kernel.util.Validator;
035 import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
036 import com.liferay.portal.util.PropsUtil;
037 import com.liferay.portal.util.PropsValues;
038 import com.liferay.portlet.documentlibrary.DuplicateFileException;
039 import com.liferay.portlet.documentlibrary.NoSuchFileException;
040
041 import java.io.File;
042 import java.io.FileInputStream;
043 import java.io.FileOutputStream;
044 import java.io.IOException;
045 import java.io.InputStream;
046 import java.io.OutputStream;
047
048 import java.util.ArrayList;
049 import java.util.Arrays;
050 import java.util.Date;
051 import java.util.List;
052 import java.util.Map.Entry;
053 import java.util.Properties;
054
055 import org.jets3t.service.Jets3tProperties;
056 import org.jets3t.service.S3Service;
057 import org.jets3t.service.S3ServiceException;
058 import org.jets3t.service.ServiceException;
059 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
060 import org.jets3t.service.model.S3Bucket;
061 import org.jets3t.service.model.S3Object;
062 import org.jets3t.service.model.StorageObject;
063 import org.jets3t.service.security.AWSCredentials;
064 import org.jets3t.service.utils.MultipartUtils;
065
066
073 public class S3Store extends BaseStore {
074
075 public S3Store() {
076 try {
077 _s3Service = getS3Service();
078 _s3Bucket = getS3Bucket();
079 }
080 catch (S3ServiceException s3se) {
081 _log.error(s3se.getMessage());
082 }
083 }
084
085 @Override
086 public void addDirectory(
087 long companyId, long repositoryId, String dirName) {
088 }
089
090 @Override
091 public void addFile(
092 long companyId, long repositoryId, String fileName, InputStream is)
093 throws SystemException {
094
095 try {
096 File file = FileUtil.createTempFile(is);
097
098 addFile(companyId, repositoryId, fileName, file);
099 }
100 catch (IOException ioe) {
101 throw new SystemException(ioe);
102 }
103 finally {
104 StreamUtil.cleanUp(is);
105 }
106 }
107
108 @Override
109 public void addFile(
110 long companyId, long repositoryId, String fileName, File file)
111 throws SystemException {
112
113 putObject(
114 getKey(companyId, repositoryId, fileName, VERSION_DEFAULT), file);
115 }
116
117 @Override
118 public void checkRoot(long companyId) {
119 }
120
121 @Override
122 public void deleteDirectory(
123 long companyId, long repositoryId, String dirName)
124 throws SystemException {
125
126 try {
127 S3Object[] s3Objects = _s3Service.listObjects(
128 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
129 null);
130
131 for (S3Object s3Object : s3Objects) {
132 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
133 }
134 }
135 catch (S3ServiceException s3se) {
136 throw new SystemException(s3se);
137 }
138 }
139
140 @Override
141 public void deleteFile(long companyId, long repositoryId, String fileName)
142 throws SystemException {
143
144 try {
145 S3Object[] s3Objects = _s3Service.listObjects(
146 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
147 null);
148
149 for (S3Object s3Object : s3Objects) {
150 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
151 }
152 }
153 catch (S3ServiceException s3se) {
154 throw new SystemException(s3se);
155 }
156 }
157
158 @Override
159 public void deleteFile(
160 long companyId, long repositoryId, String fileName,
161 String versionLabel)
162 throws SystemException {
163
164 try {
165 _s3Service.deleteObject(
166 _s3Bucket,
167 getKey(companyId, repositoryId, fileName, versionLabel));
168 }
169 catch (S3ServiceException s3se) {
170 throw new SystemException(s3se);
171 }
172 }
173
174 @Override
175 public File getFile(
176 long companyId, long repositoryId, String fileName,
177 String versionLabel)
178 throws PortalException, SystemException {
179
180 try {
181 if (Validator.isNull(versionLabel)) {
182 versionLabel = getHeadVersionLabel(
183 companyId, repositoryId, fileName);
184 }
185
186 S3Object s3Object = _s3Service.getObject(
187 _s3Bucket.getName(),
188 getKey(companyId, repositoryId, fileName, versionLabel));
189
190 File tempFile = getTempFile(s3Object, fileName);
191
192 cleanUpTempFiles();
193
194 return tempFile;
195 }
196 catch (IOException ioe) {
197 throw new SystemException(ioe);
198 }
199 catch (ServiceException se) {
200 throw new SystemException(se);
201 }
202 }
203
204 @Override
205 public InputStream getFileAsStream(
206 long companyId, long repositoryId, String fileName,
207 String versionLabel)
208 throws PortalException, SystemException {
209
210 try {
211 if (Validator.isNull(versionLabel)) {
212 versionLabel = getHeadVersionLabel(
213 companyId, repositoryId, fileName);
214 }
215
216 S3Object s3Object = _s3Service.getObject(
217 _s3Bucket.getName(),
218 getKey(companyId, repositoryId, fileName, versionLabel));
219
220 return s3Object.getDataInputStream();
221 }
222 catch (ServiceException se) {
223 throw new SystemException(se);
224 }
225 }
226
227 @Override
228 public String[] getFileNames(long companyId, long repositoryId)
229 throws SystemException {
230
231 try {
232 S3Object[] s3Objects = _s3Service.listObjects(
233 _s3Bucket.getName(), getKey(companyId, repositoryId), null);
234
235 return getFileNames(s3Objects);
236 }
237 catch (S3ServiceException s3se) {
238 throw new SystemException(s3se);
239 }
240 }
241
242 @Override
243 public String[] getFileNames(
244 long companyId, long repositoryId, String dirName)
245 throws SystemException {
246
247 try {
248 S3Object[] s3Objects = _s3Service.listObjects(
249 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
250 null);
251
252 return getFileNames(s3Objects);
253 }
254 catch (S3ServiceException s3se) {
255 throw new SystemException(s3se);
256 }
257 }
258
259 @Override
260 public long getFileSize(long companyId, long repositoryId, String fileName)
261 throws PortalException, SystemException {
262
263 try {
264 String versionLabel = getHeadVersionLabel(
265 companyId, repositoryId, fileName);
266
267 StorageObject storageObject = _s3Service.getObjectDetails(
268 _s3Bucket.getName(),
269 getKey(companyId, repositoryId, fileName, versionLabel));
270
271 return storageObject.getContentLength();
272 }
273 catch (ServiceException se) {
274 throw new SystemException(se);
275 }
276 }
277
278 @Override
279 public boolean hasDirectory(
280 long companyId, long repositoryId, String dirName) {
281
282 return true;
283 }
284
285 @Override
286 public boolean hasFile(
287 long companyId, long repositoryId, String fileName,
288 String versionLabel)
289 throws SystemException {
290
291 try {
292 S3Object[] s3Objects = _s3Service.listObjects(
293 _s3Bucket.getName(),
294 getKey(companyId, repositoryId, fileName, versionLabel), null);
295
296 if (s3Objects.length == 0) {
297 return false;
298 }
299 else {
300 return true;
301 }
302 }
303 catch (S3ServiceException s3se) {
304 throw new SystemException(s3se);
305 }
306 }
307
308 @Override
309 public void move(String srcDir, String destDir) {
310 }
311
312 @Override
313 public void updateFile(
314 long companyId, long repositoryId, long newRepositoryId,
315 String fileName)
316 throws PortalException, SystemException {
317
318 File tempFile = null;
319 InputStream is = null;
320 S3Object newS3Object = null;
321
322 if (repositoryId == newRepositoryId) {
323 throw new DuplicateFileException(
324 String.format(
325 "{companyId=%s, fileName=%s, repositoryId=%s}", companyId,
326 fileName, repositoryId));
327 }
328
329 try {
330 S3Object[] s3Objects = _s3Service.listObjects(
331 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
332 null);
333
334 for (S3Object oldS3Object : s3Objects) {
335 String oldKey = oldS3Object.getKey();
336
337 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
338
339 tempFile = new File(
340 SystemProperties.get(SystemProperties.TMP_DIR) +
341 File.separator + PortalUUIDUtil.generate());
342
343 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
344
345 is = new FileInputStream(tempFile);
346
347 String newPrefix = getKey(companyId, newRepositoryId);
348
349 int x = oldKey.indexOf(CharPool.SLASH);
350
351 x = oldKey.indexOf(CharPool.SLASH, x + 1);
352
353 String newKey = newPrefix + oldKey.substring(x);
354
355 putObject(newKey, tempFile);
356
357 _s3Service.deleteObject(_s3Bucket, oldKey);
358 }
359 }
360 catch (IOException ioe) {
361 throw new SystemException(ioe);
362 }
363 catch (ServiceException se) {
364 throw new SystemException(se);
365 }
366 finally {
367 StreamUtil.cleanUp(is);
368
369 FileUtil.delete(tempFile);
370 }
371 }
372
373 @Override
374 public void updateFile(
375 long companyId, long repositoryId, String fileName,
376 String newFileName)
377 throws PortalException, SystemException {
378
379 if (fileName.equals(newFileName)) {
380 throw new DuplicateFileException(
381 String.format(
382 "{companyId=%s, fileName=%s, repositoryId=%s}", companyId,
383 fileName, repositoryId));
384 }
385
386 File tempFile = null;
387 InputStream is = null;
388 S3Object newS3Object = null;
389
390 try {
391 S3Object[] s3Objects = _s3Service.listObjects(
392 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
393 null);
394
395 for (S3Object oldS3Object : s3Objects) {
396 String oldKey = oldS3Object.getKey();
397
398 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
399
400 tempFile = new File(
401 SystemProperties.get(SystemProperties.TMP_DIR) +
402 File.separator + PortalUUIDUtil.generate());
403
404 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
405
406 oldS3Object.closeDataInputStream();
407
408 is = new FileInputStream(tempFile);
409
410 String newPrefix = getKey(companyId, repositoryId, newFileName);
411
412 int x = oldKey.indexOf(StringPool.SLASH);
413
414 x = oldKey.indexOf(CharPool.SLASH, x + 1);
415 x = oldKey.indexOf(CharPool.SLASH, x + 1);
416
417 String newKey = newPrefix + oldKey.substring(x);
418
419 putObject(newKey, tempFile);
420
421 _s3Service.deleteObject(_s3Bucket, oldKey);
422 }
423 }
424 catch (IOException ioe) {
425 throw new SystemException(ioe);
426 }
427 catch (ServiceException se) {
428 throw new SystemException(se);
429 }
430 finally {
431 StreamUtil.cleanUp(is);
432
433 FileUtil.delete(tempFile);
434 }
435 }
436
437 @Override
438 public void updateFile(
439 long companyId, long repositoryId, String fileName,
440 String versionLabel, InputStream is)
441 throws SystemException {
442
443 File file = null;
444
445 try {
446 file = FileUtil.createTempFile(is);
447
448 updateFile(companyId, repositoryId, fileName, versionLabel, file);
449 }
450 catch (Exception e) {
451 throw new SystemException(e);
452 }
453 finally {
454 StreamUtil.cleanUp(is);
455
456 FileUtil.delete(file);
457 }
458 }
459
460 @Override
461 public void updateFile(
462 long companyId, long repositoryId, String fileName,
463 String versionLabel, File file)
464 throws PortalException, SystemException {
465
466 putObject(
467 getKey(companyId, repositoryId, fileName, versionLabel), file);
468 }
469
470 protected void cleanUpTempFiles() {
471 _calledGetFileCount++;
472
473 if (_calledGetFileCount <
474 PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
475
476 return;
477 }
478
479 synchronized (this) {
480 if (_calledGetFileCount == 0) {
481 return;
482 }
483
484 _calledGetFileCount = 0;
485
486 String tempDirName =
487 SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
488
489 File tempDir = new File(tempDirName);
490
491 long lastModified = System.currentTimeMillis();
492
493 lastModified -=
494 (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
495
496 cleanUpTempFiles(tempDir, lastModified);
497 }
498 }
499
500 protected void cleanUpTempFiles(File file, long lastModified) {
501 if (!file.isDirectory()) {
502 return;
503 }
504
505 String[] fileNames = FileUtil.listDirs(file);
506
507 if (fileNames.length == 0) {
508 if (file.lastModified() < lastModified) {
509 FileUtil.deltree(file);
510
511 return;
512 }
513 }
514 else {
515 for (String fileName : fileNames) {
516 cleanUpTempFiles(new File(file, fileName), lastModified);
517 }
518
519 String[] subfileNames = file.list();
520
521 if (subfileNames.length == 0) {
522 FileUtil.deltree(file);
523
524 return;
525 }
526 }
527 }
528
529 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
530 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
531 throw new S3ServiceException(
532 "S3 access and secret keys are not set");
533 }
534 else {
535 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
536 }
537 }
538
539 protected String getFileName(String key) {
540
541
542
543
544 int x = key.indexOf(CharPool.SLASH);
545
546 x = key.indexOf(CharPool.SLASH, x + 1);
547
548 int y = key.lastIndexOf(CharPool.SLASH);
549
550 return key.substring(x, y);
551 }
552
553 protected String[] getFileNames(S3Object[] s3Objects) {
554 List<String> fileNames = new ArrayList<String>();
555
556 for (S3Object s3Object : s3Objects) {
557 String fileName = getFileName(s3Object.getKey());
558
559 fileNames.add(fileName);
560 }
561
562 return fileNames.toArray(new String[fileNames.size()]);
563 }
564
565 protected String getHeadVersionLabel(
566 long companyId, long repositoryId, String fileName)
567 throws PortalException, S3ServiceException {
568
569 S3Object[] s3Objects = _s3Service.listObjects(
570 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
571 null);
572
573 String[] keys = new String[s3Objects.length];
574
575 for (int i = 0; i < s3Objects.length; i++) {
576 S3Object s3Object = s3Objects[i];
577
578 keys[i] = s3Object.getKey();
579 }
580
581 if (keys.length > 0) {
582 Arrays.sort(keys);
583
584 String headKey = keys[keys.length - 1];
585
586 int x = headKey.lastIndexOf(CharPool.SLASH);
587
588 return headKey.substring(x + 1);
589 }
590 else {
591 throw new NoSuchFileException(fileName);
592 }
593 }
594
595 protected Jets3tProperties getJets3tProperties() {
596 Jets3tProperties jets3tProperties = new Jets3tProperties();
597
598 Properties properties = new Properties();
599
600 for (Entry<Object, Object> entry : _jets3tProperties.entrySet()) {
601 String key = (String)entry.getKey();
602 String value = (String)entry.getValue();
603
604 properties.put(StringUtil.strip(key, _BRACKETS), value);
605 }
606
607 jets3tProperties.loadAndReplaceProperties(properties, "liferay");
608
609 if (_log.isInfoEnabled()) {
610 _log.info("Jets3t properties: " + jets3tProperties.getProperties());
611 }
612
613 return jets3tProperties;
614 }
615
616 protected String getKey(long companyId, long repositoryId) {
617 StringBundler sb = new StringBundler(4);
618
619 sb.append(companyId);
620 sb.append(StringPool.SLASH);
621 sb.append(repositoryId);
622
623 return sb.toString();
624 }
625
626 protected String getKey(
627 long companyId, long repositoryId, String fileName) {
628
629 StringBundler sb = new StringBundler(4);
630
631 sb.append(companyId);
632 sb.append(StringPool.SLASH);
633 sb.append(repositoryId);
634 sb.append(getNormalizedFileName(fileName));
635
636 return sb.toString();
637 }
638
639 protected String getKey(
640 long companyId, long repositoryId, String fileName,
641 String versionLabel) {
642
643 StringBundler sb = new StringBundler(6);
644
645 sb.append(companyId);
646 sb.append(StringPool.SLASH);
647 sb.append(repositoryId);
648 sb.append(getNormalizedFileName(fileName));
649 sb.append(StringPool.SLASH);
650 sb.append(versionLabel);
651
652 return sb.toString();
653 }
654
655 protected String getNormalizedFileName(String fileName) {
656 String normalizedFileName = fileName;
657
658 if (!fileName.startsWith(StringPool.SLASH)) {
659 normalizedFileName = StringPool.SLASH + normalizedFileName;
660 }
661
662 if (fileName.endsWith(StringPool.SLASH)) {
663 normalizedFileName = normalizedFileName.substring(
664 0, normalizedFileName.length() - 1);
665 }
666
667 return normalizedFileName;
668 }
669
670 protected S3Bucket getS3Bucket() throws S3ServiceException {
671 if (Validator.isNull(_BUCKET_NAME)) {
672 throw new S3ServiceException("S3 bucket name is not set");
673 }
674 else {
675 return getS3Service().getBucket(_BUCKET_NAME);
676 }
677 }
678
679 protected S3Service getS3Service() throws S3ServiceException {
680 AWSCredentials credentials = getAWSCredentials();
681
682 Jets3tProperties jets3tProperties = getJets3tProperties();
683
684 return new RestS3Service(
685 credentials, ReleaseInfo.getServerInfo(), null, jets3tProperties);
686 }
687
688 protected File getTempFile(S3Object s3Object, String fileName)
689 throws IOException, ServiceException {
690
691 StringBundler sb = new StringBundler(5);
692
693 sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
694 sb.append(_TEMP_DIR_NAME);
695 sb.append(
696 DateUtil.getCurrentDate(
697 _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
698 sb.append(getNormalizedFileName(fileName));
699
700 Date lastModifiedDate = s3Object.getLastModifiedDate();
701
702 sb.append(lastModifiedDate.getTime());
703
704 String tempFileName = sb.toString();
705
706 File tempFile = new File(tempFileName);
707
708 InputStream inputStream = s3Object.getDataInputStream();
709
710 if (tempFile.exists() &&
711 (tempFile.lastModified() >= lastModifiedDate.getTime())) {
712
713 StreamUtil.cleanUp(inputStream);
714
715 return tempFile;
716 }
717
718 if (inputStream == null) {
719 throw new IOException("S3 object input stream is null");
720 }
721
722 OutputStream outputStream = null;
723
724 try {
725 File parentFile = tempFile.getParentFile();
726
727 FileUtil.mkdirs(parentFile);
728
729 outputStream = new FileOutputStream(tempFile);
730
731 StreamUtil.transfer(inputStream, outputStream);
732 }
733 finally {
734 StreamUtil.cleanUp(inputStream, outputStream);
735 }
736
737 return tempFile;
738 }
739
740 protected void putObject(String key, File file) throws SystemException {
741 try {
742 MultipartUtils multipartUtils = new MultipartUtils(_PART_SIZE);
743
744 List<StorageObject> s3Objects = new ArrayList<StorageObject>();
745
746 S3Object s3Object = new S3Object(file);
747
748 s3Object.setBucketName(_BUCKET_NAME);
749 s3Object.setKey(key);
750
751 s3Objects.add(s3Object);
752
753 multipartUtils.uploadObjects(
754 _BUCKET_NAME, _s3Service, s3Objects, null);
755 }
756 catch (Exception e) {
757 throw new SystemException(e);
758 }
759 }
760
761 private static final String _ACCESS_KEY = PropsUtil.get(
762 PropsKeys.DL_STORE_S3_ACCESS_KEY);
763
764 private static char[] _BRACKETS =
765 {CharPool.CLOSE_BRACKET, CharPool.OPEN_BRACKET};
766
767 private static final String _BUCKET_NAME = PropsUtil.get(
768 PropsKeys.DL_STORE_S3_BUCKET_NAME);
769
770 private static final long _PART_SIZE = GetterUtil.getLong(
771 PropsUtil.get(PropsKeys.DL_STORE_S3_PART_SIZE));
772
773 private static final String _SECRET_KEY = PropsUtil.get(
774 PropsKeys.DL_STORE_S3_SECRET_KEY);
775
776 private static final String _TEMP_DIR_NAME = "/liferay/s3";
777
778 private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
779
780 private static Log _log = LogFactoryUtil.getLog(S3Store.class);
781
782 private static final Properties _jets3tProperties = PropsUtil.getProperties(
783 PropsKeys.DL_STORE_S3_JETS3T, true);
784
785 private int _calledGetFileCount;
786 private S3Bucket _s3Bucket;
787 private S3Service _s3Service;
788
789 }