001
014
015 package com.liferay.portlet.documentlibrary.store;
016
017 import com.liferay.portal.kernel.exception.PortalException;
018 import com.liferay.portal.kernel.exception.SystemException;
019 import com.liferay.portal.kernel.log.Log;
020 import com.liferay.portal.kernel.log.LogFactoryUtil;
021 import com.liferay.portal.kernel.util.CharPool;
022 import com.liferay.portal.kernel.util.DateUtil;
023 import com.liferay.portal.kernel.util.FileUtil;
024 import com.liferay.portal.kernel.util.GetterUtil;
025 import com.liferay.portal.kernel.util.LocaleUtil;
026 import com.liferay.portal.kernel.util.PropsKeys;
027 import com.liferay.portal.kernel.util.ReleaseInfo;
028 import com.liferay.portal.kernel.util.StreamUtil;
029 import com.liferay.portal.kernel.util.StringBundler;
030 import com.liferay.portal.kernel.util.StringPool;
031 import com.liferay.portal.kernel.util.SystemProperties;
032 import com.liferay.portal.kernel.util.Time;
033 import com.liferay.portal.kernel.util.Validator;
034 import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
035 import com.liferay.portal.util.PropsUtil;
036 import com.liferay.portal.util.PropsValues;
037 import com.liferay.portlet.documentlibrary.DuplicateFileException;
038 import com.liferay.portlet.documentlibrary.NoSuchFileException;
039
040 import java.io.File;
041 import java.io.FileInputStream;
042 import java.io.FileOutputStream;
043 import java.io.IOException;
044 import java.io.InputStream;
045 import java.io.OutputStream;
046
047 import java.util.ArrayList;
048 import java.util.Arrays;
049 import java.util.Date;
050 import java.util.List;
051 import java.util.Properties;
052
053 import org.jets3t.service.Jets3tProperties;
054 import org.jets3t.service.S3Service;
055 import org.jets3t.service.S3ServiceException;
056 import org.jets3t.service.ServiceException;
057 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
058 import org.jets3t.service.model.S3Bucket;
059 import org.jets3t.service.model.S3Object;
060 import org.jets3t.service.model.StorageObject;
061 import org.jets3t.service.security.AWSCredentials;
062 import org.jets3t.service.utils.MultipartUtils;
063
064
071 public class S3Store extends BaseStore {
072
073 public S3Store() {
074 try {
075 _s3Service = getS3Service();
076 _s3Bucket = getS3Bucket();
077 }
078 catch (S3ServiceException s3se) {
079 _log.error(s3se.getMessage());
080 }
081 }
082
083 @Override
084 public void addDirectory(
085 long companyId, long repositoryId, String dirName) {
086 }
087
088 @Override
089 public void addFile(
090 long companyId, long repositoryId, String fileName, InputStream is)
091 throws SystemException {
092
093 try {
094 File file = FileUtil.createTempFile(is);
095
096 addFile(companyId, repositoryId, fileName, file);
097 }
098 catch (IOException ioe) {
099 throw new SystemException(ioe);
100 }
101 finally {
102 StreamUtil.cleanUp(is);
103 }
104 }
105
106 @Override
107 public void addFile(
108 long companyId, long repositoryId, String fileName, File file)
109 throws SystemException {
110
111 putObject(
112 getKey(companyId, repositoryId, fileName, VERSION_DEFAULT), file);
113 }
114
115 @Override
116 public void checkRoot(long companyId) {
117 }
118
119 @Override
120 public void deleteDirectory(
121 long companyId, long repositoryId, String dirName)
122 throws SystemException {
123
124 try {
125 S3Object[] s3Objects = _s3Service.listObjects(
126 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
127 null);
128
129 for (S3Object s3Object : s3Objects) {
130 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
131 }
132 }
133 catch (S3ServiceException s3se) {
134 throw new SystemException(s3se);
135 }
136 }
137
138 @Override
139 public void deleteFile(long companyId, long repositoryId, String fileName)
140 throws SystemException {
141
142 try {
143 S3Object[] s3Objects = _s3Service.listObjects(
144 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
145 null);
146
147 for (S3Object s3Object : s3Objects) {
148 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
149 }
150 }
151 catch (S3ServiceException s3se) {
152 throw new SystemException(s3se);
153 }
154 }
155
156 @Override
157 public void deleteFile(
158 long companyId, long repositoryId, String fileName,
159 String versionLabel)
160 throws SystemException {
161
162 try {
163 _s3Service.deleteObject(
164 _s3Bucket,
165 getKey(companyId, repositoryId, fileName, versionLabel));
166 }
167 catch (S3ServiceException s3se) {
168 throw new SystemException(s3se);
169 }
170 }
171
172 @Override
173 public File getFile(
174 long companyId, long repositoryId, String fileName,
175 String versionLabel)
176 throws PortalException, SystemException {
177
178 try {
179 if (Validator.isNull(versionLabel)) {
180 versionLabel = getHeadVersionLabel(
181 companyId, repositoryId, fileName);
182 }
183
184 S3Object s3Object = _s3Service.getObject(
185 _s3Bucket.getName(),
186 getKey(companyId, repositoryId, fileName, versionLabel));
187
188 File tempFile = getTempFile(s3Object, fileName);
189
190 cleanUpTempFiles();
191
192 return tempFile;
193 }
194 catch (IOException ioe) {
195 throw new SystemException(ioe);
196 }
197 catch (ServiceException se) {
198 throw new SystemException(se);
199 }
200 }
201
202 @Override
203 public InputStream getFileAsStream(
204 long companyId, long repositoryId, String fileName,
205 String versionLabel)
206 throws PortalException, SystemException {
207
208 try {
209 if (Validator.isNull(versionLabel)) {
210 versionLabel = getHeadVersionLabel(
211 companyId, repositoryId, fileName);
212 }
213
214 S3Object s3Object = _s3Service.getObject(
215 _s3Bucket.getName(),
216 getKey(companyId, repositoryId, fileName, versionLabel));
217
218 return s3Object.getDataInputStream();
219 }
220 catch (ServiceException se) {
221 throw new SystemException(se);
222 }
223 }
224
225 @Override
226 public String[] getFileNames(long companyId, long repositoryId)
227 throws SystemException {
228
229 try {
230 S3Object[] s3Objects = _s3Service.listObjects(
231 _s3Bucket.getName(), getKey(companyId, repositoryId), null);
232
233 return getFileNames(s3Objects);
234 }
235 catch (S3ServiceException s3se) {
236 throw new SystemException(s3se);
237 }
238 }
239
240 @Override
241 public String[] getFileNames(
242 long companyId, long repositoryId, String dirName)
243 throws SystemException {
244
245 try {
246 S3Object[] s3Objects = _s3Service.listObjects(
247 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
248 null);
249
250 return getFileNames(s3Objects);
251 }
252 catch (S3ServiceException s3se) {
253 throw new SystemException(s3se);
254 }
255 }
256
257 @Override
258 public long getFileSize(long companyId, long repositoryId, String fileName)
259 throws PortalException, SystemException {
260
261 try {
262 String versionLabel = getHeadVersionLabel(
263 companyId, repositoryId, fileName);
264
265 StorageObject storageObject = _s3Service.getObjectDetails(
266 _s3Bucket.getName(),
267 getKey(companyId, repositoryId, fileName, versionLabel));
268
269 return storageObject.getContentLength();
270 }
271 catch (ServiceException se) {
272 throw new SystemException(se);
273 }
274 }
275
276 @Override
277 public boolean hasDirectory(
278 long companyId, long repositoryId, String dirName) {
279
280 return true;
281 }
282
283 @Override
284 public boolean hasFile(
285 long companyId, long repositoryId, String fileName,
286 String versionLabel)
287 throws SystemException {
288
289 try {
290 S3Object[] s3Objects = _s3Service.listObjects(
291 _s3Bucket.getName(),
292 getKey(companyId, repositoryId, fileName, versionLabel), null);
293
294 if (s3Objects.length == 0) {
295 return false;
296 }
297 else {
298 return true;
299 }
300 }
301 catch (S3ServiceException s3se) {
302 throw new SystemException(s3se);
303 }
304 }
305
306 @Override
307 public void move(String srcDir, String destDir) {
308 }
309
310 @Override
311 public void updateFile(
312 long companyId, long repositoryId, long newRepositoryId,
313 String fileName)
314 throws PortalException, SystemException {
315
316 File tempFile = null;
317 InputStream is = null;
318 S3Object newS3Object = null;
319
320 if (repositoryId == newRepositoryId) {
321 throw new DuplicateFileException(
322 String.format(
323 "{companyId=%s, fileName=%s, repositoryId=%s}", companyId,
324 fileName, repositoryId));
325 }
326
327 try {
328 S3Object[] s3Objects = _s3Service.listObjects(
329 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
330 null);
331
332 for (S3Object oldS3Object : s3Objects) {
333 String oldKey = oldS3Object.getKey();
334
335 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
336
337 tempFile = new File(
338 SystemProperties.get(SystemProperties.TMP_DIR) +
339 File.separator + PortalUUIDUtil.generate());
340
341 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
342
343 is = new FileInputStream(tempFile);
344
345 String newPrefix = getKey(companyId, newRepositoryId);
346
347 int x = oldKey.indexOf(CharPool.SLASH);
348
349 x = oldKey.indexOf(CharPool.SLASH, x + 1);
350
351 String newKey = newPrefix + oldKey.substring(x);
352
353 putObject(newKey, tempFile);
354
355 _s3Service.deleteObject(_s3Bucket, oldKey);
356 }
357 }
358 catch (IOException ioe) {
359 throw new SystemException(ioe);
360 }
361 catch (ServiceException se) {
362 throw new SystemException(se);
363 }
364 finally {
365 StreamUtil.cleanUp(is);
366
367 FileUtil.delete(tempFile);
368 }
369 }
370
371 @Override
372 public void updateFile(
373 long companyId, long repositoryId, String fileName,
374 String newFileName)
375 throws PortalException, SystemException {
376
377 if (fileName.equals(newFileName)) {
378 throw new DuplicateFileException(
379 String.format(
380 "{companyId=%s, fileName=%s, repositoryId=%s}", companyId,
381 fileName, repositoryId));
382 }
383
384 File tempFile = null;
385 InputStream is = null;
386 S3Object newS3Object = null;
387
388 try {
389 S3Object[] s3Objects = _s3Service.listObjects(
390 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
391 null);
392
393 for (S3Object oldS3Object : s3Objects) {
394 String oldKey = oldS3Object.getKey();
395
396 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
397
398 tempFile = new File(
399 SystemProperties.get(SystemProperties.TMP_DIR) +
400 File.separator + PortalUUIDUtil.generate());
401
402 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
403
404 oldS3Object.closeDataInputStream();
405
406 is = new FileInputStream(tempFile);
407
408 String newPrefix = getKey(companyId, repositoryId, newFileName);
409
410 int x = oldKey.indexOf(StringPool.SLASH);
411
412 x = oldKey.indexOf(CharPool.SLASH, x + 1);
413 x = oldKey.indexOf(CharPool.SLASH, x + 1);
414
415 String newKey = newPrefix + oldKey.substring(x);
416
417 putObject(newKey, tempFile);
418
419 _s3Service.deleteObject(_s3Bucket, oldKey);
420 }
421 }
422 catch (IOException ioe) {
423 throw new SystemException(ioe);
424 }
425 catch (ServiceException se) {
426 throw new SystemException(se);
427 }
428 finally {
429 StreamUtil.cleanUp(is);
430
431 FileUtil.delete(tempFile);
432 }
433 }
434
435 @Override
436 public void updateFile(
437 long companyId, long repositoryId, String fileName,
438 String versionLabel, InputStream is)
439 throws SystemException {
440
441 File file = null;
442
443 try {
444 file = FileUtil.createTempFile(is);
445
446 updateFile(companyId, repositoryId, fileName, versionLabel, file);
447 }
448 catch (Exception e) {
449 throw new SystemException(e);
450 }
451 finally {
452 StreamUtil.cleanUp(is);
453
454 FileUtil.delete(file);
455 }
456 }
457
458 @Override
459 public void updateFile(
460 long companyId, long repositoryId, String fileName,
461 String versionLabel, File file)
462 throws PortalException, SystemException {
463
464 putObject(
465 getKey(companyId, repositoryId, fileName, versionLabel), file);
466 }
467
468 protected void cleanUpTempFiles() {
469 _calledGetFileCount++;
470
471 if (_calledGetFileCount <
472 PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
473
474 return;
475 }
476
477 synchronized (this) {
478 if (_calledGetFileCount == 0) {
479 return;
480 }
481
482 _calledGetFileCount = 0;
483
484 String tempDirName =
485 SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
486
487 File tempDir = new File(tempDirName);
488
489 long lastModified = System.currentTimeMillis();
490
491 lastModified -=
492 (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
493
494 cleanUpTempFiles(tempDir, lastModified);
495 }
496 }
497
498 protected void cleanUpTempFiles(File file, long lastModified) {
499 if (!file.isDirectory()) {
500 return;
501 }
502
503 String[] fileNames = FileUtil.listDirs(file);
504
505 if (fileNames.length == 0) {
506 if (file.lastModified() < lastModified) {
507 FileUtil.deltree(file);
508
509 return;
510 }
511 }
512 else {
513 for (String fileName : fileNames) {
514 cleanUpTempFiles(new File(file, fileName), lastModified);
515 }
516
517 String[] subfileNames = file.list();
518
519 if (subfileNames.length == 0) {
520 FileUtil.deltree(file);
521
522 return;
523 }
524 }
525 }
526
527 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
528 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
529 throw new S3ServiceException(
530 "S3 access and secret keys are not set");
531 }
532 else {
533 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
534 }
535 }
536
537 protected String getFileName(String key) {
538
539
540
541
542 int x = key.indexOf(CharPool.SLASH);
543
544 x = key.indexOf(CharPool.SLASH, x + 1);
545
546 int y = key.lastIndexOf(CharPool.SLASH);
547
548 return key.substring(x, y);
549 }
550
551 protected String[] getFileNames(S3Object[] s3Objects) {
552 List<String> fileNames = new ArrayList<String>();
553
554 for (S3Object s3Object : s3Objects) {
555 String fileName = getFileName(s3Object.getKey());
556
557 fileNames.add(fileName);
558 }
559
560 return fileNames.toArray(new String[fileNames.size()]);
561 }
562
563 protected String getHeadVersionLabel(
564 long companyId, long repositoryId, String fileName)
565 throws PortalException, S3ServiceException {
566
567 S3Object[] s3Objects = _s3Service.listObjects(
568 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
569 null);
570
571 String[] keys = new String[s3Objects.length];
572
573 for (int i = 0; i < s3Objects.length; i++) {
574 S3Object s3Object = s3Objects[i];
575
576 keys[i] = s3Object.getKey();
577 }
578
579 if (keys.length > 0) {
580 Arrays.sort(keys);
581
582 String headKey = keys[keys.length - 1];
583
584 int x = headKey.lastIndexOf(CharPool.SLASH);
585
586 return headKey.substring(x + 1);
587 }
588 else {
589 throw new NoSuchFileException(fileName);
590 }
591 }
592
593 protected Jets3tProperties getJets3tProperties() {
594 Jets3tProperties jets3tProperties = new Jets3tProperties();
595
596 jets3tProperties.loadAndReplaceProperties(_jets3tProperties, "liferay");
597
598 if (_log.isInfoEnabled()) {
599 _log.info("Jets3t properties: " + jets3tProperties.getProperties());
600 }
601
602 return jets3tProperties;
603 }
604
605 protected String getKey(long companyId, long repositoryId) {
606 StringBundler sb = new StringBundler(4);
607
608 sb.append(companyId);
609 sb.append(StringPool.SLASH);
610 sb.append(repositoryId);
611
612 return sb.toString();
613 }
614
615 protected String getKey(
616 long companyId, long repositoryId, String fileName) {
617
618 StringBundler sb = new StringBundler(4);
619
620 sb.append(companyId);
621 sb.append(StringPool.SLASH);
622 sb.append(repositoryId);
623 sb.append(getNormalizedFileName(fileName));
624
625 return sb.toString();
626 }
627
628 protected String getKey(
629 long companyId, long repositoryId, String fileName,
630 String versionLabel) {
631
632 StringBundler sb = new StringBundler(6);
633
634 sb.append(companyId);
635 sb.append(StringPool.SLASH);
636 sb.append(repositoryId);
637 sb.append(getNormalizedFileName(fileName));
638 sb.append(StringPool.SLASH);
639 sb.append(versionLabel);
640
641 return sb.toString();
642 }
643
644 protected String getNormalizedFileName(String fileName) {
645 String normalizedFileName = fileName;
646
647 if (!fileName.startsWith(StringPool.SLASH)) {
648 normalizedFileName = StringPool.SLASH + normalizedFileName;
649 }
650
651 if (fileName.endsWith(StringPool.SLASH)) {
652 normalizedFileName = normalizedFileName.substring(
653 0, normalizedFileName.length() - 1);
654 }
655
656 return normalizedFileName;
657 }
658
659 protected S3Bucket getS3Bucket() throws S3ServiceException {
660 if (Validator.isNull(_BUCKET_NAME)) {
661 throw new S3ServiceException("S3 bucket name is not set");
662 }
663 else {
664 return getS3Service().getBucket(_BUCKET_NAME);
665 }
666 }
667
668 protected S3Service getS3Service() throws S3ServiceException {
669 AWSCredentials credentials = getAWSCredentials();
670
671 Jets3tProperties jets3tProperties = getJets3tProperties();
672
673 return new RestS3Service(
674 credentials, ReleaseInfo.getServerInfo(), null, jets3tProperties);
675 }
676
677 protected File getTempFile(S3Object s3Object, String fileName)
678 throws IOException, ServiceException {
679
680 StringBundler sb = new StringBundler(5);
681
682 sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
683 sb.append(_TEMP_DIR_NAME);
684 sb.append(
685 DateUtil.getCurrentDate(
686 _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
687 sb.append(getNormalizedFileName(fileName));
688
689 Date lastModifiedDate = s3Object.getLastModifiedDate();
690
691 sb.append(lastModifiedDate.getTime());
692
693 String tempFileName = sb.toString();
694
695 File tempFile = new File(tempFileName);
696
697 InputStream inputStream = s3Object.getDataInputStream();
698
699 if (tempFile.exists() &&
700 (tempFile.lastModified() >= lastModifiedDate.getTime())) {
701
702 StreamUtil.cleanUp(inputStream);
703
704 return tempFile;
705 }
706
707 if (inputStream == null) {
708 throw new IOException("S3 object input stream is null");
709 }
710
711 OutputStream outputStream = null;
712
713 try {
714 File parentFile = tempFile.getParentFile();
715
716 FileUtil.mkdirs(parentFile);
717
718 outputStream = new FileOutputStream(tempFile);
719
720 StreamUtil.transfer(inputStream, outputStream);
721 }
722 finally {
723 StreamUtil.cleanUp(inputStream, outputStream);
724 }
725
726 return tempFile;
727 }
728
729 protected void putObject(String key, File file) throws SystemException {
730 try {
731 MultipartUtils multipartUtils = new MultipartUtils(_PART_SIZE);
732
733 List<StorageObject> s3Objects = new ArrayList<StorageObject>();
734
735 S3Object s3Object = new S3Object(file);
736
737 s3Object.setBucketName(_BUCKET_NAME);
738 s3Object.setKey(key);
739
740 s3Objects.add(s3Object);
741
742 multipartUtils.uploadObjects(
743 _BUCKET_NAME, _s3Service, s3Objects, null);
744 }
745 catch (Exception e) {
746 throw new SystemException(e);
747 }
748 }
749
750 private static final String _ACCESS_KEY = PropsUtil.get(
751 PropsKeys.DL_STORE_S3_ACCESS_KEY);
752
753 private static final String _BUCKET_NAME = PropsUtil.get(
754 PropsKeys.DL_STORE_S3_BUCKET_NAME);
755
756 private static final long _PART_SIZE = GetterUtil.getLong(
757 PropsUtil.get(PropsKeys.DL_STORE_S3_PART_SIZE));
758
759 private static final String _SECRET_KEY = PropsUtil.get(
760 PropsKeys.DL_STORE_S3_SECRET_KEY);
761
762 private static final String _TEMP_DIR_NAME = "/liferay/s3";
763
764 private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
765
766 private static Log _log = LogFactoryUtil.getLog(S3Store.class);
767
768 private static final Properties _jets3tProperties = PropsUtil.getProperties(
769 PropsKeys.DL_STORE_S3_JETS3T, true);
770
771 private int _calledGetFileCount;
772 private S3Bucket _s3Bucket;
773 private S3Service _s3Service;
774
775 }