001
014
015 package com.liferay.portlet.documentlibrary.store;
016
017 import com.liferay.portal.kernel.exception.SystemException;
018 import com.liferay.portal.kernel.log.Log;
019 import com.liferay.portal.kernel.log.LogFactoryUtil;
020 import com.liferay.portal.kernel.util.CharPool;
021 import com.liferay.portal.kernel.util.DateUtil;
022 import com.liferay.portal.kernel.util.FileUtil;
023 import com.liferay.portal.kernel.util.LocaleUtil;
024 import com.liferay.portal.kernel.util.PropsKeys;
025 import com.liferay.portal.kernel.util.ReleaseInfo;
026 import com.liferay.portal.kernel.util.StreamUtil;
027 import com.liferay.portal.kernel.util.StringBundler;
028 import com.liferay.portal.kernel.util.StringPool;
029 import com.liferay.portal.kernel.util.SystemProperties;
030 import com.liferay.portal.kernel.util.Time;
031 import com.liferay.portal.kernel.util.Validator;
032 import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033 import com.liferay.portal.util.PropsUtil;
034 import com.liferay.portal.util.PropsValues;
035 import com.liferay.portlet.documentlibrary.DuplicateFileException;
036 import com.liferay.portlet.documentlibrary.NoSuchFileException;
037
038 import java.io.File;
039 import java.io.FileInputStream;
040 import java.io.FileOutputStream;
041 import java.io.IOException;
042 import java.io.InputStream;
043 import java.io.OutputStream;
044
045 import java.util.ArrayList;
046 import java.util.Arrays;
047 import java.util.Date;
048 import java.util.List;
049 import java.util.Properties;
050
051 import org.jets3t.service.Jets3tProperties;
052 import org.jets3t.service.S3Service;
053 import org.jets3t.service.S3ServiceException;
054 import org.jets3t.service.ServiceException;
055 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
056 import org.jets3t.service.model.S3Bucket;
057 import org.jets3t.service.model.S3Object;
058 import org.jets3t.service.model.StorageObject;
059 import org.jets3t.service.security.AWSCredentials;
060
061
068 public class S3Store extends BaseStore {
069
070 public S3Store() {
071 S3Service s3Service = null;
072 S3Bucket s3Bucket = null;
073
074 try {
075 s3Service = getS3Service();
076 s3Bucket = getS3Bucket();
077 }
078 catch (S3ServiceException s3se) {
079 _log.error(s3se.getMessage());
080 }
081
082 _s3Service = s3Service;
083 _s3Bucket = s3Bucket;
084 }
085
086 @Override
087 public void addDirectory(
088 long companyId, long repositoryId, String dirName) {
089 }
090
091 @Override
092 public void addFile(
093 long companyId, long repositoryId, String fileName, InputStream is)
094 throws DuplicateFileException {
095
096 if (hasFile(companyId, repositoryId, fileName)) {
097 throw new DuplicateFileException(companyId, repositoryId, fileName);
098 }
099
100 try {
101 S3Object s3Object = new S3Object(
102 _s3Bucket,
103 getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
104
105 s3Object.setDataInputStream(is);
106
107 _s3Service.putObject(_s3Bucket, s3Object);
108 }
109 catch (S3ServiceException s3se) {
110 throw new SystemException(s3se);
111 }
112 finally {
113 StreamUtil.cleanUp(is);
114 }
115 }
116
117 @Override
118 public void checkRoot(long companyId) {
119 }
120
121 @Override
122 public void deleteDirectory(
123 long companyId, long repositoryId, String dirName) {
124
125 try {
126 S3Object[] s3Objects = _s3Service.listObjects(
127 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
128 null);
129
130 for (S3Object s3Object : s3Objects) {
131 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
132 }
133 }
134 catch (S3ServiceException s3se) {
135 if (isS3NoSuchKeyException(s3se)) {
136 logFailedDeletion(companyId, repositoryId, dirName);
137 }
138 else {
139 throw new SystemException(s3se);
140 }
141 }
142 }
143
144 @Override
145 public void deleteFile(long companyId, long repositoryId, String fileName) {
146 try {
147 S3Object[] s3Objects = _s3Service.listObjects(
148 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
149 null);
150
151 for (S3Object s3Object : s3Objects) {
152 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
153 }
154 }
155 catch (S3ServiceException s3se) {
156 if (isS3NoSuchKeyException(s3se)) {
157 logFailedDeletion(companyId, repositoryId, fileName);
158 }
159 else {
160 throw new SystemException(s3se);
161 }
162 }
163 }
164
165 @Override
166 public void deleteFile(
167 long companyId, long repositoryId, String fileName,
168 String versionLabel) {
169
170 try {
171 _s3Service.deleteObject(
172 _s3Bucket,
173 getKey(companyId, repositoryId, fileName, versionLabel));
174 }
175 catch (S3ServiceException s3se) {
176 if (isS3NoSuchKeyException(s3se)) {
177 logFailedDeletion(companyId, repositoryId, fileName);
178 }
179 else {
180 throw new SystemException(s3se);
181 }
182 }
183 }
184
185 @Override
186 public File getFile(
187 long companyId, long repositoryId, String fileName,
188 String versionLabel)
189 throws NoSuchFileException {
190
191 try {
192 if (Validator.isNull(versionLabel)) {
193 versionLabel = getHeadVersionLabel(
194 companyId, repositoryId, fileName);
195 }
196
197 S3Object s3Object = _s3Service.getObject(
198 _s3Bucket.getName(),
199 getKey(companyId, repositoryId, fileName, versionLabel));
200
201 File tempFile = getTempFile(s3Object, fileName);
202
203 cleanUpTempFiles();
204
205 return tempFile;
206 }
207 catch (Exception e) {
208 if (isS3NoSuchKeyException(e)) {
209 throw new NoSuchFileException(
210 companyId, repositoryId, fileName, versionLabel, e);
211 }
212
213 throw new SystemException(e);
214 }
215 }
216
217 @Override
218 public InputStream getFileAsStream(
219 long companyId, long repositoryId, String fileName,
220 String versionLabel)
221 throws NoSuchFileException {
222
223 try {
224 if (Validator.isNull(versionLabel)) {
225 versionLabel = getHeadVersionLabel(
226 companyId, repositoryId, fileName);
227 }
228
229 S3Object s3Object = _s3Service.getObject(
230 _s3Bucket.getName(),
231 getKey(companyId, repositoryId, fileName, versionLabel));
232
233 return s3Object.getDataInputStream();
234 }
235 catch (ServiceException se) {
236 if (isS3NoSuchKeyException(se)) {
237 throw new NoSuchFileException(
238 companyId, repositoryId, fileName, versionLabel, se);
239 }
240
241 throw new SystemException(se);
242 }
243 }
244
245 @Override
246 public String[] getFileNames(long companyId, long repositoryId) {
247 try {
248 S3Object[] s3Objects = _s3Service.listObjects(
249 _s3Bucket.getName(), getKey(companyId, repositoryId), null);
250
251 return getFileNames(s3Objects);
252 }
253 catch (S3ServiceException s3se) {
254 if (isS3NoSuchKeyException(s3se)) {
255 return new String[0];
256 }
257
258 throw new SystemException(s3se);
259 }
260 }
261
262 @Override
263 public String[] getFileNames(
264 long companyId, long repositoryId, String dirName) {
265
266 try {
267 S3Object[] s3Objects = _s3Service.listObjects(
268 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
269 null);
270
271 return getFileNames(s3Objects);
272 }
273 catch (S3ServiceException s3se) {
274 if (isS3NoSuchKeyException(s3se)) {
275 return new String[0];
276 }
277
278 throw new SystemException(s3se);
279 }
280 }
281
282 @Override
283 public long getFileSize(long companyId, long repositoryId, String fileName)
284 throws NoSuchFileException {
285
286 try {
287 String versionLabel = getHeadVersionLabel(
288 companyId, repositoryId, fileName);
289
290 StorageObject storageObject = _s3Service.getObjectDetails(
291 _s3Bucket.getName(),
292 getKey(companyId, repositoryId, fileName, versionLabel));
293
294 return storageObject.getContentLength();
295 }
296 catch (ServiceException se) {
297 throw new SystemException(se);
298 }
299 }
300
301 @Override
302 public boolean hasDirectory(
303 long companyId, long repositoryId, String dirName) {
304
305 return true;
306 }
307
308 @Override
309 public boolean hasFile(
310 long companyId, long repositoryId, String fileName,
311 String versionLabel) {
312
313 try {
314 S3Object[] s3Objects = _s3Service.listObjects(
315 _s3Bucket.getName(),
316 getKey(companyId, repositoryId, fileName, versionLabel), null);
317
318 if (s3Objects.length == 0) {
319 return false;
320 }
321 else {
322 return true;
323 }
324 }
325 catch (S3ServiceException s3se) {
326 if (isS3NoSuchKeyException(s3se)) {
327 return false;
328 }
329
330 throw new SystemException(s3se);
331 }
332 }
333
334 @Override
335 public void updateFile(
336 long companyId, long repositoryId, long newRepositoryId,
337 String fileName)
338 throws DuplicateFileException, NoSuchFileException {
339
340 File tempFile = null;
341 InputStream is = null;
342 S3Object newS3Object = null;
343
344 if (!hasFile(companyId, repositoryId, fileName)) {
345 throw new NoSuchFileException(companyId, newRepositoryId, fileName);
346 }
347
348 if (hasFile(companyId, newRepositoryId, fileName)) {
349 throw new DuplicateFileException(
350 companyId, newRepositoryId, fileName);
351 }
352
353 try {
354 S3Object[] s3Objects = _s3Service.listObjects(
355 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
356 null);
357
358 for (S3Object oldS3Object : s3Objects) {
359 String oldKey = oldS3Object.getKey();
360
361 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
362
363 tempFile = new File(
364 SystemProperties.get(SystemProperties.TMP_DIR) +
365 File.separator + PortalUUIDUtil.generate());
366
367 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
368
369 is = new FileInputStream(tempFile);
370
371 String newPrefix = getKey(companyId, newRepositoryId);
372
373 int x = oldKey.indexOf(CharPool.SLASH);
374
375 x = oldKey.indexOf(CharPool.SLASH, x + 1);
376
377 String newKey = newPrefix + oldKey.substring(x);
378
379 newS3Object = new S3Object(_s3Bucket, newKey);
380
381 newS3Object.setDataInputStream(is);
382
383 _s3Service.putObject(_s3Bucket, newS3Object);
384 _s3Service.deleteObject(_s3Bucket, oldKey);
385 }
386 }
387 catch (Exception e) {
388 throw new SystemException(e);
389 }
390 finally {
391 StreamUtil.cleanUp(is);
392
393 FileUtil.delete(tempFile);
394 }
395 }
396
397 @Override
398 public void updateFile(
399 long companyId, long repositoryId, String fileName,
400 String newFileName)
401 throws DuplicateFileException, NoSuchFileException {
402
403 File tempFile = null;
404 InputStream is = null;
405 S3Object newS3Object = null;
406
407 if (!hasFile(companyId, repositoryId, fileName)) {
408 throw new NoSuchFileException(companyId, repositoryId, fileName);
409 }
410
411 if (hasFile(companyId, repositoryId, newFileName)) {
412 throw new DuplicateFileException(
413 companyId, repositoryId, newFileName);
414 }
415
416 try {
417 S3Object[] s3Objects = _s3Service.listObjects(
418 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
419 null);
420
421 for (S3Object oldS3Object : s3Objects) {
422 String oldKey = oldS3Object.getKey();
423
424 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
425
426 tempFile = new File(
427 SystemProperties.get(SystemProperties.TMP_DIR) +
428 File.separator + PortalUUIDUtil.generate());
429
430 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
431
432 oldS3Object.closeDataInputStream();
433
434 is = new FileInputStream(tempFile);
435
436 String newPrefix = getKey(companyId, repositoryId, newFileName);
437
438 int x = oldKey.indexOf(StringPool.SLASH);
439
440 x = oldKey.indexOf(CharPool.SLASH, x + 1);
441 x = oldKey.indexOf(CharPool.SLASH, x + 1);
442
443 String newKey = newPrefix + oldKey.substring(x);
444
445 newS3Object = new S3Object(_s3Bucket, newKey);
446
447 newS3Object.setDataInputStream(is);
448
449 _s3Service.putObject(_s3Bucket, newS3Object);
450 _s3Service.deleteObject(_s3Bucket, oldKey);
451 }
452 }
453 catch (Exception e) {
454 throw new SystemException(e);
455 }
456 finally {
457 StreamUtil.cleanUp(is);
458
459 FileUtil.delete(tempFile);
460 }
461 }
462
463 @Override
464 public void updateFile(
465 long companyId, long repositoryId, String fileName,
466 String versionLabel, InputStream is)
467 throws DuplicateFileException, NoSuchFileException {
468
469 if (hasFile(companyId, repositoryId, fileName, versionLabel)) {
470 throw new DuplicateFileException(
471 companyId, repositoryId, fileName, versionLabel);
472 }
473
474 try {
475 S3Object s3Object = new S3Object(
476 _s3Bucket,
477 getKey(companyId, repositoryId, fileName, versionLabel));
478
479 s3Object.setDataInputStream(is);
480
481 _s3Service.putObject(_s3Bucket, s3Object);
482 }
483 catch (Exception e) {
484 if (isS3NoSuchKeyException(e)) {
485 throw new NoSuchFileException(
486 companyId, repositoryId, fileName, versionLabel, e);
487 }
488
489 throw new SystemException(e);
490 }
491 finally {
492 StreamUtil.cleanUp(is);
493 }
494 }
495
496 protected void cleanUpTempFiles() {
497 _calledGetFileCount++;
498
499 if (_calledGetFileCount <
500 PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
501
502 return;
503 }
504
505 synchronized (this) {
506 if (_calledGetFileCount == 0) {
507 return;
508 }
509
510 _calledGetFileCount = 0;
511
512 String tempDirName =
513 SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
514
515 File tempDir = new File(tempDirName);
516
517 long lastModified = System.currentTimeMillis();
518
519 lastModified -=
520 (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
521
522 cleanUpTempFiles(tempDir, lastModified);
523 }
524 }
525
526 protected void cleanUpTempFiles(File file, long lastModified) {
527 if (!file.isDirectory()) {
528 return;
529 }
530
531 String[] fileNames = FileUtil.listDirs(file);
532
533 if (fileNames.length == 0) {
534 if (file.lastModified() < lastModified) {
535 FileUtil.deltree(file);
536
537 return;
538 }
539 }
540 else {
541 for (String fileName : fileNames) {
542 cleanUpTempFiles(new File(file, fileName), lastModified);
543 }
544
545 String[] subfileNames = file.list();
546
547 if (subfileNames.length == 0) {
548 FileUtil.deltree(file);
549
550 return;
551 }
552 }
553 }
554
555 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
556 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
557 throw new S3ServiceException(
558 "S3 access and secret keys are not set");
559 }
560 else {
561 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
562 }
563 }
564
565 protected String getFileName(String key) {
566
567
568
569
570 int x = key.indexOf(CharPool.SLASH);
571
572 x = key.indexOf(CharPool.SLASH, x + 1);
573
574 int y = key.lastIndexOf(CharPool.SLASH);
575
576 return key.substring(x + 1, y);
577 }
578
579 protected String[] getFileNames(S3Object[] s3Objects) {
580 List<String> fileNames = new ArrayList<>(s3Objects.length);
581
582 for (S3Object s3Object : s3Objects) {
583 String fileName = getFileName(s3Object.getKey());
584
585 fileNames.add(fileName);
586 }
587
588 return fileNames.toArray(new String[fileNames.size()]);
589 }
590
591 protected String getHeadVersionLabel(
592 long companyId, long repositoryId, String fileName)
593 throws NoSuchFileException, S3ServiceException {
594
595 S3Object[] s3Objects = _s3Service.listObjects(
596 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
597 null);
598
599 String[] keys = new String[s3Objects.length];
600
601 for (int i = 0; i < s3Objects.length; i++) {
602 S3Object s3Object = s3Objects[i];
603
604 keys[i] = s3Object.getKey();
605 }
606
607 if (keys.length > 0) {
608 Arrays.sort(keys);
609
610 String headKey = keys[keys.length - 1];
611
612 int x = headKey.lastIndexOf(CharPool.SLASH);
613
614 return headKey.substring(x + 1);
615 }
616 else {
617 throw new NoSuchFileException(companyId, repositoryId, fileName);
618 }
619 }
620
621 protected Jets3tProperties getJets3tProperties() {
622 Jets3tProperties jets3tProperties = new Jets3tProperties();
623
624 jets3tProperties.loadAndReplaceProperties(_jets3tProperties, "liferay");
625
626 if (_log.isInfoEnabled()) {
627 _log.info("Jets3t properties: " + jets3tProperties.getProperties());
628 }
629
630 return jets3tProperties;
631 }
632
633 protected String getKey(long companyId, long repositoryId) {
634 return companyId + StringPool.SLASH + repositoryId;
635 }
636
637 protected String getKey(
638 long companyId, long repositoryId, String fileName) {
639
640 StringBundler sb = new StringBundler(4);
641
642 sb.append(companyId);
643 sb.append(StringPool.SLASH);
644 sb.append(repositoryId);
645 sb.append(getNormalizedFileName(fileName));
646
647 return sb.toString();
648 }
649
650 protected String getKey(
651 long companyId, long repositoryId, String fileName,
652 String versionLabel) {
653
654 StringBundler sb = new StringBundler(6);
655
656 sb.append(companyId);
657 sb.append(StringPool.SLASH);
658 sb.append(repositoryId);
659 sb.append(getNormalizedFileName(fileName));
660 sb.append(StringPool.SLASH);
661 sb.append(versionLabel);
662
663 return sb.toString();
664 }
665
666 protected String getNormalizedFileName(String fileName) {
667 String normalizedFileName = fileName;
668
669 if (!fileName.startsWith(StringPool.SLASH)) {
670 normalizedFileName = StringPool.SLASH + normalizedFileName;
671 }
672
673 if (fileName.endsWith(StringPool.SLASH)) {
674 normalizedFileName = normalizedFileName.substring(
675 0, normalizedFileName.length() - 1);
676 }
677
678 return normalizedFileName;
679 }
680
681 protected S3Bucket getS3Bucket() throws S3ServiceException {
682 if (Validator.isNull(_BUCKET_NAME)) {
683 throw new S3ServiceException("S3 bucket name is not set");
684 }
685 else {
686 return getS3Service().getBucket(_BUCKET_NAME);
687 }
688 }
689
690 protected S3Service getS3Service() throws S3ServiceException {
691 AWSCredentials credentials = getAWSCredentials();
692
693 Jets3tProperties jets3tProperties = getJets3tProperties();
694
695 return new RestS3Service(
696 credentials, ReleaseInfo.getServerInfo(), null, jets3tProperties);
697 }
698
699 protected File getTempFile(S3Object s3Object, String fileName)
700 throws IOException, ServiceException {
701
702 StringBundler sb = new StringBundler(5);
703
704 sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
705 sb.append(_TEMP_DIR_NAME);
706 sb.append(
707 DateUtil.getCurrentDate(
708 _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
709 sb.append(getNormalizedFileName(fileName));
710
711 Date lastModifiedDate = s3Object.getLastModifiedDate();
712
713 sb.append(lastModifiedDate.getTime());
714
715 String tempFileName = sb.toString();
716
717 File tempFile = new File(tempFileName);
718
719 if (tempFile.exists() &&
720 (tempFile.lastModified() >= lastModifiedDate.getTime())) {
721
722 return tempFile;
723 }
724
725 InputStream inputStream = s3Object.getDataInputStream();
726
727 if (inputStream == null) {
728 throw new IOException("S3 object input stream is null");
729 }
730
731 OutputStream outputStream = null;
732
733 try {
734 File parentFile = tempFile.getParentFile();
735
736 FileUtil.mkdirs(parentFile);
737
738 outputStream = new FileOutputStream(tempFile);
739
740 StreamUtil.transfer(inputStream, outputStream);
741 }
742 finally {
743 StreamUtil.cleanUp(inputStream, outputStream);
744 }
745
746 return tempFile;
747 }
748
749 protected boolean isS3NoSuchKeyException(Exception e) {
750 if (e instanceof S3ServiceException) {
751 S3ServiceException s3se = (S3ServiceException)e;
752
753 if (_S3_NO_SUCH_KEY_ERROR_CODE.equals(s3se.getS3ErrorCode())) {
754 return true;
755 }
756 }
757
758 return false;
759 }
760
761 private static final String _ACCESS_KEY = PropsUtil.get(
762 PropsKeys.DL_STORE_S3_ACCESS_KEY);
763
764 private static final String _BUCKET_NAME = PropsUtil.get(
765 PropsKeys.DL_STORE_S3_BUCKET_NAME);
766
767 private static final String _S3_NO_SUCH_KEY_ERROR_CODE = "NoSuchKey";
768
769 private static final String _SECRET_KEY = PropsUtil.get(
770 PropsKeys.DL_STORE_S3_SECRET_KEY);
771
772 private static final String _TEMP_DIR_NAME = "/liferay/s3";
773
774 private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
775
776 private static final Log _log = LogFactoryUtil.getLog(S3Store.class);
777
778 private static final Properties _jets3tProperties = PropsUtil.getProperties(
779 PropsKeys.DL_STORE_S3_JETS3T, true);
780
781 private int _calledGetFileCount;
782 private final S3Bucket _s3Bucket;
783 private final S3Service _s3Service;
784
785 }