001
014
015 package com.liferay.portlet.documentlibrary.store;
016
017 import com.liferay.portal.kernel.exception.PortalException;
018 import com.liferay.portal.kernel.exception.SystemException;
019 import com.liferay.portal.kernel.log.Log;
020 import com.liferay.portal.kernel.log.LogFactoryUtil;
021 import com.liferay.portal.kernel.util.CharPool;
022 import com.liferay.portal.kernel.util.DateUtil;
023 import com.liferay.portal.kernel.util.FileUtil;
024 import com.liferay.portal.kernel.util.GetterUtil;
025 import com.liferay.portal.kernel.util.LocaleUtil;
026 import com.liferay.portal.kernel.util.PropsKeys;
027 import com.liferay.portal.kernel.util.StreamUtil;
028 import com.liferay.portal.kernel.util.StringBundler;
029 import com.liferay.portal.kernel.util.StringPool;
030 import com.liferay.portal.kernel.util.SystemProperties;
031 import com.liferay.portal.kernel.util.Time;
032 import com.liferay.portal.kernel.util.Validator;
033 import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
034 import com.liferay.portal.util.PropsUtil;
035 import com.liferay.portal.util.PropsValues;
036 import com.liferay.portlet.documentlibrary.NoSuchFileException;
037
038 import java.io.File;
039 import java.io.FileInputStream;
040 import java.io.FileOutputStream;
041 import java.io.IOException;
042 import java.io.InputStream;
043 import java.io.OutputStream;
044
045 import java.util.ArrayList;
046 import java.util.Arrays;
047 import java.util.Date;
048 import java.util.List;
049
050 import org.jets3t.service.S3Service;
051 import org.jets3t.service.S3ServiceException;
052 import org.jets3t.service.ServiceException;
053 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
054 import org.jets3t.service.model.S3Bucket;
055 import org.jets3t.service.model.S3Object;
056 import org.jets3t.service.model.StorageObject;
057 import org.jets3t.service.security.AWSCredentials;
058 import org.jets3t.service.utils.MultipartUtils;
059
060
067 public class S3Store extends BaseStore {
068
069 public S3Store() {
070 try {
071 _s3Service = getS3Service();
072 _s3Bucket = getS3Bucket();
073 }
074 catch (S3ServiceException s3se) {
075 _log.error(s3se.getMessage());
076 }
077 }
078
079 @Override
080 public void addDirectory(
081 long companyId, long repositoryId, String dirName) {
082 }
083
084 @Override
085 public void addFile(
086 long companyId, long repositoryId, String fileName, InputStream is)
087 throws SystemException {
088
089 try {
090 File file = FileUtil.createTempFile(is);
091
092 addFile(companyId, repositoryId, fileName, file);
093 }
094 catch (IOException ioe) {
095 throw new SystemException(ioe);
096 }
097 finally {
098 StreamUtil.cleanUp(is);
099 }
100 }
101
102 @Override
103 public void addFile(
104 long companyId, long repositoryId, String fileName, File file)
105 throws SystemException {
106
107 putObject(
108 getKey(companyId, repositoryId, fileName, VERSION_DEFAULT), file);
109 }
110
111 @Override
112 public void checkRoot(long companyId) {
113 }
114
115 @Override
116 public void deleteDirectory(
117 long companyId, long repositoryId, String dirName)
118 throws SystemException {
119
120 try {
121 S3Object[] s3Objects = _s3Service.listObjects(
122 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
123 null);
124
125 for (S3Object s3Object : s3Objects) {
126 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
127 }
128 }
129 catch (S3ServiceException s3se) {
130 throw new SystemException(s3se);
131 }
132 }
133
134 @Override
135 public void deleteFile(long companyId, long repositoryId, String fileName)
136 throws SystemException {
137
138 try {
139 S3Object[] s3Objects = _s3Service.listObjects(
140 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
141 null);
142
143 for (S3Object s3Object : s3Objects) {
144 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
145 }
146 }
147 catch (S3ServiceException s3se) {
148 throw new SystemException(s3se);
149 }
150 }
151
152 @Override
153 public void deleteFile(
154 long companyId, long repositoryId, String fileName,
155 String versionLabel)
156 throws SystemException {
157
158 try {
159 _s3Service.deleteObject(
160 _s3Bucket,
161 getKey(companyId, repositoryId, fileName, versionLabel));
162 }
163 catch (S3ServiceException s3se) {
164 throw new SystemException(s3se);
165 }
166 }
167
168 @Override
169 public File getFile(
170 long companyId, long repositoryId, String fileName,
171 String versionLabel)
172 throws PortalException, SystemException {
173
174 try {
175 if (Validator.isNull(versionLabel)) {
176 versionLabel = getHeadVersionLabel(
177 companyId, repositoryId, fileName);
178 }
179
180 S3Object s3Object = _s3Service.getObject(
181 _s3Bucket.getName(),
182 getKey(companyId, repositoryId, fileName, versionLabel));
183
184 File tempFile = getTempFile(s3Object, fileName);
185
186 cleanUpTempFiles();
187
188 return tempFile;
189 }
190 catch (IOException ioe) {
191 throw new SystemException(ioe);
192 }
193 catch (ServiceException se) {
194 throw new SystemException(se);
195 }
196 }
197
198 @Override
199 public InputStream getFileAsStream(
200 long companyId, long repositoryId, String fileName,
201 String versionLabel)
202 throws PortalException, SystemException {
203
204 try {
205 if (Validator.isNull(versionLabel)) {
206 versionLabel = getHeadVersionLabel(
207 companyId, repositoryId, fileName);
208 }
209
210 S3Object s3Object = _s3Service.getObject(
211 _s3Bucket.getName(),
212 getKey(companyId, repositoryId, fileName, versionLabel));
213
214 return s3Object.getDataInputStream();
215 }
216 catch (ServiceException se) {
217 throw new SystemException(se);
218 }
219 }
220
221 @Override
222 public String[] getFileNames(long companyId, long repositoryId)
223 throws SystemException {
224
225 try {
226 S3Object[] s3Objects = _s3Service.listObjects(
227 _s3Bucket.getName(), getKey(companyId, repositoryId), null);
228
229 return getFileNames(s3Objects);
230 }
231 catch (S3ServiceException s3se) {
232 throw new SystemException(s3se);
233 }
234 }
235
236 @Override
237 public String[] getFileNames(
238 long companyId, long repositoryId, String dirName)
239 throws SystemException {
240
241 try {
242 S3Object[] s3Objects = _s3Service.listObjects(
243 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
244 null);
245
246 return getFileNames(s3Objects);
247 }
248 catch (S3ServiceException s3se) {
249 throw new SystemException(s3se);
250 }
251 }
252
253 @Override
254 public long getFileSize(long companyId, long repositoryId, String fileName)
255 throws PortalException, SystemException {
256
257 try {
258 String versionLabel = getHeadVersionLabel(
259 companyId, repositoryId, fileName);
260
261 StorageObject storageObject = _s3Service.getObjectDetails(
262 _s3Bucket.getName(),
263 getKey(companyId, repositoryId, fileName, versionLabel));
264
265 return storageObject.getContentLength();
266 }
267 catch (ServiceException se) {
268 throw new SystemException(se);
269 }
270 }
271
272 @Override
273 public boolean hasDirectory(
274 long companyId, long repositoryId, String dirName) {
275
276 return true;
277 }
278
279 @Override
280 public boolean hasFile(
281 long companyId, long repositoryId, String fileName,
282 String versionLabel)
283 throws SystemException {
284
285 try {
286 S3Object[] s3Objects = _s3Service.listObjects(
287 _s3Bucket.getName(),
288 getKey(companyId, repositoryId, fileName, versionLabel), null);
289
290 if (s3Objects.length == 0) {
291 return false;
292 }
293 else {
294 return true;
295 }
296 }
297 catch (S3ServiceException s3se) {
298 throw new SystemException(s3se);
299 }
300 }
301
302 @Override
303 public void move(String srcDir, String destDir) {
304 }
305
306 @Override
307 public void updateFile(
308 long companyId, long repositoryId, long newRepositoryId,
309 String fileName)
310 throws SystemException {
311
312 File tempFile = null;
313 InputStream is = null;
314 S3Object newS3Object = null;
315
316 try {
317 S3Object[] s3Objects = _s3Service.listObjects(
318 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
319 null);
320
321 for (S3Object oldS3Object : s3Objects) {
322 String oldKey = oldS3Object.getKey();
323
324 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
325
326 tempFile = new File(
327 SystemProperties.get(SystemProperties.TMP_DIR) +
328 File.separator + PortalUUIDUtil.generate());
329
330 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
331
332 is = new FileInputStream(tempFile);
333
334 String newPrefix = getKey(companyId, newRepositoryId);
335
336 int x = oldKey.indexOf(CharPool.SLASH);
337
338 x = oldKey.indexOf(CharPool.SLASH, x + 1);
339
340 String newKey = newPrefix + oldKey.substring(x);
341
342 putObject(newKey, tempFile);
343
344 _s3Service.deleteObject(_s3Bucket, oldKey);
345 }
346 }
347 catch (IOException ioe) {
348 throw new SystemException(ioe);
349 }
350 catch (ServiceException se) {
351 throw new SystemException(se);
352 }
353 finally {
354 StreamUtil.cleanUp(is);
355
356 FileUtil.delete(tempFile);
357 }
358 }
359
360 @Override
361 public void updateFile(
362 long companyId, long repositoryId, String fileName,
363 String newFileName)
364 throws SystemException {
365
366 File tempFile = null;
367 InputStream is = null;
368 S3Object newS3Object = null;
369
370 try {
371 S3Object[] s3Objects = _s3Service.listObjects(
372 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
373 null);
374
375 for (S3Object oldS3Object : s3Objects) {
376 String oldKey = oldS3Object.getKey();
377
378 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
379
380 tempFile = new File(
381 SystemProperties.get(SystemProperties.TMP_DIR) +
382 File.separator + PortalUUIDUtil.generate());
383
384 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
385
386 oldS3Object.closeDataInputStream();
387
388 is = new FileInputStream(tempFile);
389
390 String newPrefix = getKey(companyId, repositoryId, newFileName);
391
392 int x = oldKey.indexOf(StringPool.SLASH);
393
394 x = oldKey.indexOf(CharPool.SLASH, x + 1);
395 x = oldKey.indexOf(CharPool.SLASH, x + 1);
396
397 String newKey = newPrefix + oldKey.substring(x);
398
399 putObject(newKey, tempFile);
400
401 _s3Service.deleteObject(_s3Bucket, oldKey);
402 }
403 }
404 catch (IOException ioe) {
405 throw new SystemException(ioe);
406 }
407 catch (ServiceException se) {
408 throw new SystemException(se);
409 }
410 finally {
411 StreamUtil.cleanUp(is);
412
413 FileUtil.delete(tempFile);
414 }
415 }
416
417 @Override
418 public void updateFile(
419 long companyId, long repositoryId, String fileName,
420 String versionLabel, InputStream is)
421 throws SystemException {
422
423 try {
424 File file = FileUtil.createTempFile(is);
425
426 updateFile(companyId, repositoryId, fileName, versionLabel, file);
427 }
428 catch (Exception e) {
429 throw new SystemException(e);
430 }
431 finally {
432 StreamUtil.cleanUp(is);
433 }
434 }
435
436 @Override
437 public void updateFile(
438 long companyId, long repositoryId, String fileName,
439 String versionLabel, File file)
440 throws PortalException, SystemException {
441
442 putObject(
443 getKey(companyId, repositoryId, fileName, versionLabel), file);
444 }
445
446 protected void cleanUpTempFiles() {
447 _calledGetFileCount++;
448
449 if (_calledGetFileCount <
450 PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
451
452 return;
453 }
454
455 synchronized (this) {
456 if (_calledGetFileCount == 0) {
457 return;
458 }
459
460 _calledGetFileCount = 0;
461
462 String tempDirName =
463 SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
464
465 File tempDir = new File(tempDirName);
466
467 long lastModified = System.currentTimeMillis();
468
469 lastModified -=
470 (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
471
472 cleanUpTempFiles(tempDir, lastModified);
473 }
474 }
475
476 protected void cleanUpTempFiles(File file, long lastModified) {
477 if (!file.isDirectory()) {
478 return;
479 }
480
481 String[] fileNames = FileUtil.listDirs(file);
482
483 if (fileNames.length == 0) {
484 if (file.lastModified() < lastModified) {
485 FileUtil.deltree(file);
486
487 return;
488 }
489 }
490 else {
491 for (String fileName : fileNames) {
492 cleanUpTempFiles(new File(file, fileName), lastModified);
493 }
494
495 String[] subfileNames = file.list();
496
497 if (subfileNames.length == 0) {
498 FileUtil.deltree(file);
499
500 return;
501 }
502 }
503 }
504
505 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
506 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
507 throw new S3ServiceException(
508 "S3 access and secret keys are not set");
509 }
510 else {
511 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
512 }
513 }
514
515 protected String getFileName(String key) {
516
517
518
519
520 int x = key.indexOf(CharPool.SLASH);
521
522 x = key.indexOf(CharPool.SLASH, x + 1);
523
524 int y = key.lastIndexOf(CharPool.SLASH);
525
526 return key.substring(x, y);
527 }
528
529 protected String[] getFileNames(S3Object[] s3Objects) {
530 List<String> fileNames = new ArrayList<String>();
531
532 for (S3Object s3Object : s3Objects) {
533 String fileName = getFileName(s3Object.getKey());
534
535 fileNames.add(fileName);
536 }
537
538 return fileNames.toArray(new String[fileNames.size()]);
539 }
540
541 protected String getHeadVersionLabel(
542 long companyId, long repositoryId, String fileName)
543 throws PortalException, S3ServiceException {
544
545 S3Object[] s3Objects = _s3Service.listObjects(
546 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
547 null);
548
549 String[] keys = new String[s3Objects.length];
550
551 for (int i = 0; i < s3Objects.length; i++) {
552 S3Object s3Object = s3Objects[i];
553
554 keys[i] = s3Object.getKey();
555 }
556
557 if (keys.length > 0) {
558 Arrays.sort(keys);
559
560 String headKey = keys[keys.length - 1];
561
562 int x = headKey.lastIndexOf(CharPool.SLASH);
563
564 return headKey.substring(x + 1);
565 }
566 else {
567 throw new NoSuchFileException(fileName);
568 }
569 }
570
571 protected String getKey(long companyId, long repositoryId) {
572 StringBundler sb = new StringBundler(4);
573
574 sb.append(companyId);
575 sb.append(StringPool.SLASH);
576 sb.append(repositoryId);
577
578 return sb.toString();
579 }
580
581 protected String getKey(
582 long companyId, long repositoryId, String fileName) {
583
584 StringBundler sb = new StringBundler(4);
585
586 sb.append(companyId);
587 sb.append(StringPool.SLASH);
588 sb.append(repositoryId);
589 sb.append(getNormalizedFileName(fileName));
590
591 return sb.toString();
592 }
593
594 protected String getKey(
595 long companyId, long repositoryId, String fileName,
596 String versionLabel) {
597
598 StringBundler sb = new StringBundler(6);
599
600 sb.append(companyId);
601 sb.append(StringPool.SLASH);
602 sb.append(repositoryId);
603 sb.append(getNormalizedFileName(fileName));
604 sb.append(StringPool.SLASH);
605 sb.append(versionLabel);
606
607 return sb.toString();
608 }
609
610 protected String getNormalizedFileName(String fileName) {
611 String normalizedFileName = fileName;
612
613 if (!fileName.startsWith(StringPool.SLASH)) {
614 normalizedFileName = StringPool.SLASH + normalizedFileName;
615 }
616
617 if (fileName.endsWith(StringPool.SLASH)) {
618 normalizedFileName = normalizedFileName.substring(
619 0, normalizedFileName.length() - 1);
620 }
621
622 return normalizedFileName;
623 }
624
625 protected S3Bucket getS3Bucket() throws S3ServiceException {
626 if (Validator.isNull(_BUCKET_NAME)) {
627 throw new S3ServiceException("S3 bucket name is not set");
628 }
629 else {
630 return getS3Service().getBucket(_BUCKET_NAME);
631 }
632 }
633
634 protected S3Service getS3Service() throws S3ServiceException {
635 AWSCredentials credentials = getAWSCredentials();
636
637 return new RestS3Service(credentials);
638 }
639
640 protected File getTempFile(S3Object s3Object, String fileName)
641 throws IOException, ServiceException {
642
643 StringBundler sb = new StringBundler(5);
644
645 sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
646 sb.append(_TEMP_DIR_NAME);
647 sb.append(
648 DateUtil.getCurrentDate(
649 _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
650 sb.append(getNormalizedFileName(fileName));
651
652 Date lastModifiedDate = s3Object.getLastModifiedDate();
653
654 sb.append(lastModifiedDate.getTime());
655
656 String tempFileName = sb.toString();
657
658 File tempFile = new File(tempFileName);
659
660 InputStream inputStream = s3Object.getDataInputStream();
661
662 if (tempFile.exists() &&
663 (tempFile.lastModified() >= lastModifiedDate.getTime())) {
664
665 StreamUtil.cleanUp(inputStream);
666
667 return tempFile;
668 }
669
670 if (inputStream == null) {
671 throw new IOException("S3 object input stream is null");
672 }
673
674 OutputStream outputStream = null;
675
676 try {
677 File parentFile = tempFile.getParentFile();
678
679 FileUtil.mkdirs(parentFile);
680
681 outputStream = new FileOutputStream(tempFile);
682
683 StreamUtil.transfer(inputStream, outputStream);
684 }
685 finally {
686 StreamUtil.cleanUp(inputStream, outputStream);
687 }
688
689 return tempFile;
690 }
691
692 protected void putObject(String key, File file) throws SystemException {
693 try {
694 MultipartUtils multipartUtils = new MultipartUtils(_PART_SIZE);
695
696 List<StorageObject> s3Objects = new ArrayList<StorageObject>();
697
698 S3Object s3Object = new S3Object(file);
699
700 s3Object.setBucketName(_BUCKET_NAME);
701 s3Object.setKey(key);
702
703 s3Objects.add(s3Object);
704
705 multipartUtils.uploadObjects(
706 _BUCKET_NAME, _s3Service, s3Objects, null);
707 }
708 catch (Exception e) {
709 throw new SystemException(e);
710 }
711 }
712
713 private static final String _ACCESS_KEY = PropsUtil.get(
714 PropsKeys.DL_STORE_S3_ACCESS_KEY);
715
716 private static final String _BUCKET_NAME = PropsUtil.get(
717 PropsKeys.DL_STORE_S3_BUCKET_NAME);
718
719 private static final long _PART_SIZE = GetterUtil.getLong(
720 PropsUtil.get(PropsKeys.DL_STORE_S3_PART_SIZE));
721
722 private static final String _SECRET_KEY = PropsUtil.get(
723 PropsKeys.DL_STORE_S3_SECRET_KEY);
724
725 private static final String _TEMP_DIR_NAME = "/liferay/s3";
726
727 private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
728
729 private static Log _log = LogFactoryUtil.getLog(S3Store.class);
730
731 private int _calledGetFileCount;
732 private S3Bucket _s3Bucket;
733 private S3Service _s3Service;
734
735 }