001
014
015 package com.liferay.portlet.documentlibrary.store;
016
017 import com.liferay.portal.kernel.exception.PortalException;
018 import com.liferay.portal.kernel.exception.SystemException;
019 import com.liferay.portal.kernel.log.Log;
020 import com.liferay.portal.kernel.log.LogFactoryUtil;
021 import com.liferay.portal.kernel.util.CharPool;
022 import com.liferay.portal.kernel.util.DateUtil;
023 import com.liferay.portal.kernel.util.FileUtil;
024 import com.liferay.portal.kernel.util.LocaleUtil;
025 import com.liferay.portal.kernel.util.PropsKeys;
026 import com.liferay.portal.kernel.util.StreamUtil;
027 import com.liferay.portal.kernel.util.StringBundler;
028 import com.liferay.portal.kernel.util.StringPool;
029 import com.liferay.portal.kernel.util.SystemProperties;
030 import com.liferay.portal.kernel.util.Time;
031 import com.liferay.portal.kernel.util.Validator;
032 import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033 import com.liferay.portal.util.PropsUtil;
034 import com.liferay.portal.util.PropsValues;
035 import com.liferay.portlet.documentlibrary.NoSuchFileException;
036
037 import java.io.File;
038 import java.io.FileInputStream;
039 import java.io.FileOutputStream;
040 import java.io.IOException;
041 import java.io.InputStream;
042 import java.io.OutputStream;
043
044 import java.util.ArrayList;
045 import java.util.Arrays;
046 import java.util.Date;
047 import java.util.List;
048
049 import org.jets3t.service.S3Service;
050 import org.jets3t.service.S3ServiceException;
051 import org.jets3t.service.ServiceException;
052 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
053 import org.jets3t.service.model.S3Bucket;
054 import org.jets3t.service.model.S3Object;
055 import org.jets3t.service.model.StorageObject;
056 import org.jets3t.service.security.AWSCredentials;
057
058
065 public class S3Store extends BaseStore {
066
067 public S3Store() {
068 try {
069 _s3Service = getS3Service();
070 _s3Bucket = getS3Bucket();
071 }
072 catch (S3ServiceException s3se) {
073 _log.error(s3se.getMessage());
074 }
075 }
076
077 @Override
078 public void addDirectory(
079 long companyId, long repositoryId, String dirName) {
080 }
081
082 @Override
083 public void addFile(
084 long companyId, long repositoryId, String fileName, InputStream is)
085 throws SystemException {
086
087 try {
088 S3Object s3Object = new S3Object(
089 _s3Bucket,
090 getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
091
092 s3Object.setDataInputStream(is);
093
094 _s3Service.putObject(_s3Bucket, s3Object);
095
096 is.close();
097 }
098 catch (IOException ioe) {
099 throw new SystemException(ioe);
100 }
101 catch (S3ServiceException s3se) {
102 throw new SystemException(s3se);
103 }
104 }
105
106 @Override
107 public void checkRoot(long companyId) {
108 }
109
110 @Override
111 public void deleteDirectory(
112 long companyId, long repositoryId, String dirName)
113 throws SystemException {
114
115 try {
116 S3Object[] s3Objects = _s3Service.listObjects(
117 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
118 null);
119
120 for (int i = 0; i < s3Objects.length; i++) {
121 S3Object s3Object = s3Objects[i];
122
123 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
124 }
125 }
126 catch (S3ServiceException s3se) {
127 throw new SystemException(s3se);
128 }
129 }
130
131 @Override
132 public void deleteFile(long companyId, long repositoryId, String fileName)
133 throws SystemException {
134
135 try {
136 S3Object[] s3Objects = _s3Service.listObjects(
137 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
138 null);
139
140 for (int i = 0; i < s3Objects.length; i++) {
141 S3Object s3Object = s3Objects[i];
142
143 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
144 }
145 }
146 catch (S3ServiceException s3se) {
147 throw new SystemException(s3se);
148 }
149 }
150
151 @Override
152 public void deleteFile(
153 long companyId, long repositoryId, String fileName,
154 String versionLabel)
155 throws SystemException {
156
157 try {
158 _s3Service.deleteObject(
159 _s3Bucket,
160 getKey(companyId, repositoryId, fileName, versionLabel));
161 }
162 catch (S3ServiceException s3se) {
163 throw new SystemException(s3se);
164 }
165 }
166
167 @Override
168 public File getFile(
169 long companyId, long repositoryId, String fileName,
170 String versionLabel)
171 throws PortalException, SystemException {
172
173 try {
174 if (Validator.isNull(versionLabel)) {
175 versionLabel = getHeadVersionLabel(
176 companyId, repositoryId, fileName);
177 }
178
179 S3Object s3Object = _s3Service.getObject(
180 _s3Bucket.getName(),
181 getKey(companyId, repositoryId, fileName, versionLabel));
182
183 File tempFile = getTempFile(s3Object, fileName);
184
185 cleanUpTempFiles();
186
187 return tempFile;
188 }
189 catch (IOException ioe) {
190 throw new SystemException(ioe);
191 }
192 catch (ServiceException se) {
193 throw new SystemException(se);
194 }
195 }
196
197 @Override
198 public InputStream getFileAsStream(
199 long companyId, long repositoryId, String fileName,
200 String versionLabel)
201 throws PortalException, SystemException {
202
203 try {
204 if (Validator.isNull(versionLabel)) {
205 versionLabel = getHeadVersionLabel(
206 companyId, repositoryId, fileName);
207 }
208
209 S3Object s3Object = _s3Service.getObject(
210 _s3Bucket.getName(),
211 getKey(companyId, repositoryId, fileName, versionLabel));
212
213 return s3Object.getDataInputStream();
214 }
215 catch (ServiceException se) {
216 throw new SystemException(se);
217 }
218 }
219
220 public String[] getFileNames(long companyId, long repositoryId)
221 throws SystemException {
222
223 List<String> fileNames = new ArrayList<String>();
224
225 try {
226 S3Object[] searchObjects = _s3Service.listObjects(
227 _s3Bucket.getName(), getKey(companyId, repositoryId), null);
228
229 for (int i = 0; i < searchObjects.length; i++) {
230 S3Object currentObject = searchObjects[i];
231
232 String fileName = getFileName(currentObject.getKey());
233
234 fileNames.add(fileName);
235 }
236 }
237 catch (S3ServiceException s3se) {
238 throw new SystemException(s3se);
239 }
240
241 return fileNames.toArray(new String[fileNames.size()]);
242 }
243
244 @Override
245 public String[] getFileNames(
246 long companyId, long repositoryId, String dirName)
247 throws SystemException {
248
249 try {
250 List<String> list = new ArrayList<String>();
251
252 S3Object[] s3Objects = _s3Service.listObjects(
253 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
254 null);
255
256 for (int i = 0; i < s3Objects.length; i++) {
257 S3Object s3Object = s3Objects[i];
258
259
260
261
262 String key = s3Object.getKey();
263
264 int x = key.indexOf(CharPool.SLASH);
265
266 x = key.indexOf(CharPool.SLASH, x + 1);
267
268 int y = key.lastIndexOf(CharPool.SLASH);
269
270 list.add(key.substring(x, y));
271 }
272
273 return list.toArray(new String[list.size()]);
274 }
275 catch (S3ServiceException s3se) {
276 throw new SystemException(s3se);
277 }
278 }
279
280 @Override
281 public long getFileSize(long companyId, long repositoryId, String fileName)
282 throws PortalException, SystemException {
283
284 try {
285 String versionLabel = getHeadVersionLabel(
286 companyId, repositoryId, fileName);
287
288 StorageObject storageObject = _s3Service.getObjectDetails(
289 _s3Bucket.getName(),
290 getKey(companyId, repositoryId, fileName, versionLabel));
291
292 return storageObject.getContentLength();
293 }
294 catch (ServiceException se) {
295 throw new SystemException(se);
296 }
297 }
298
299 @Override
300 public boolean hasDirectory(
301 long companyId, long repositoryId, String dirName) {
302
303 return true;
304 }
305
306 @Override
307 public boolean hasFile(
308 long companyId, long repositoryId, String fileName,
309 String versionLabel)
310 throws SystemException {
311
312 try {
313 S3Object[] s3Objects = _s3Service.listObjects(
314 _s3Bucket.getName(),
315 getKey(companyId, repositoryId, fileName, versionLabel), null);
316
317 if (s3Objects.length == 0) {
318 return false;
319 }
320 else {
321 return true;
322 }
323 }
324 catch (S3ServiceException s3se) {
325 throw new SystemException(s3se);
326 }
327 }
328
329 @Override
330 public void move(String srcDir, String destDir) {
331 }
332
333 @Override
334 public void updateFile(
335 long companyId, long repositoryId, long newRepositoryId,
336 String fileName)
337 throws SystemException {
338
339 try {
340 S3Object[] s3Objects = _s3Service.listObjects(
341 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
342 null);
343
344 for (int i = 0; i < s3Objects.length; i++) {
345 S3Object oldS3Object = s3Objects[i];
346
347 String oldKey = oldS3Object.getKey();
348
349 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
350
351 File tempFile = new File(
352 SystemProperties.get(SystemProperties.TMP_DIR) +
353 File.separator + PortalUUIDUtil.generate());
354
355 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
356
357 InputStream is = new FileInputStream(tempFile);
358
359 String newPrefix = getKey(companyId, newRepositoryId);
360
361 int x = oldKey.indexOf(CharPool.SLASH);
362
363 x = oldKey.indexOf(CharPool.SLASH, x + 1);
364
365 String newKey = newPrefix + oldKey.substring(x + 1);
366
367 S3Object newS3Object = new S3Object(_s3Bucket, newKey);
368
369 newS3Object.setDataInputStream(is);
370
371 _s3Service.putObject(_s3Bucket, newS3Object);
372 _s3Service.deleteObject(_s3Bucket, oldKey);
373
374 is.close();
375
376 FileUtil.delete(tempFile);
377 }
378 }
379 catch (IOException ioe) {
380 throw new SystemException(ioe);
381 }
382 catch (ServiceException se) {
383 throw new SystemException(se);
384 }
385 }
386
387 public void updateFile(
388 long companyId, long repositoryId, String fileName,
389 String newFileName)
390 throws SystemException {
391
392 try {
393 S3Object[] s3Objects = _s3Service.listObjects(
394 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
395 null);
396
397 for (int i = 0; i < s3Objects.length; i++) {
398 S3Object oldS3Object = s3Objects[i];
399
400 String oldKey = oldS3Object.getKey();
401
402 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
403
404 File tempFile = new File(
405 SystemProperties.get(SystemProperties.TMP_DIR) +
406 File.separator + PortalUUIDUtil.generate());
407
408 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
409
410 oldS3Object.closeDataInputStream();
411
412 InputStream is = new FileInputStream(tempFile);
413
414 String newPrefix = getKey(companyId, repositoryId, newFileName);
415
416 int x = oldKey.indexOf(StringPool.SLASH);
417
418 x = oldKey.indexOf(CharPool.SLASH, x + 1);
419 x = oldKey.indexOf(CharPool.SLASH, x + 1);
420
421 String newKey = newPrefix + oldKey.substring(x + 1);
422
423 S3Object newS3Object = new S3Object(_s3Bucket, newKey);
424
425 newS3Object.setDataInputStream(is);
426
427 _s3Service.putObject(_s3Bucket, newS3Object);
428 _s3Service.deleteObject(_s3Bucket, oldKey);
429
430 newS3Object.closeDataInputStream();
431
432 is.close();
433
434 FileUtil.delete(tempFile);
435 }
436 }
437 catch (IOException ioe) {
438 throw new SystemException(ioe);
439 }
440 catch (ServiceException se) {
441 throw new SystemException(se);
442 }
443 }
444
445 @Override
446 public void updateFile(
447 long companyId, long repositoryId, String fileName,
448 String versionLabel, InputStream is)
449 throws SystemException {
450
451 try {
452 S3Object s3Object = new S3Object(
453 _s3Bucket,
454 getKey(companyId, repositoryId, fileName, versionLabel));
455
456 s3Object.setDataInputStream(is);
457
458 _s3Service.putObject(_s3Bucket, s3Object);
459
460 is.close();
461 }
462 catch (IOException ioe) {
463 throw new SystemException(ioe);
464 }
465 catch (S3ServiceException s3se) {
466 throw new SystemException(s3se);
467 }
468 }
469
470 protected void cleanUpTempFiles() {
471 _calledGetFileCount++;
472
473 if (_calledGetFileCount <
474 PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
475
476 return;
477 }
478
479 synchronized (this) {
480 if (_calledGetFileCount == 0) {
481 return;
482 }
483
484 _calledGetFileCount = 0;
485
486 String tempDirName =
487 SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
488
489 File tempDir = new File(tempDirName);
490
491 long lastModified = System.currentTimeMillis();
492
493 lastModified -=
494 (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
495
496 cleanUpTempFiles(tempDir, lastModified);
497 }
498 }
499
500 protected void cleanUpTempFiles(File file, long lastModified) {
501 if (!file.isDirectory()) {
502 return;
503 }
504
505 String[] fileNames = FileUtil.listDirs(file);
506
507 if (fileNames.length == 0) {
508 if (file.lastModified() < lastModified) {
509 FileUtil.deltree(file);
510
511 return;
512 }
513 }
514 else {
515 for (String fileName : fileNames) {
516 cleanUpTempFiles(new File(file, fileName), lastModified);
517 }
518
519 String[] subfileNames = file.list();
520
521 if (subfileNames.length == 0) {
522 FileUtil.deltree(file);
523
524 return;
525 }
526 }
527 }
528
529 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
530 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
531 throw new S3ServiceException(
532 "S3 access and secret keys are not set");
533 }
534 else {
535 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
536 }
537 }
538
539 protected String getFileName(String key) {
540 int x = key.indexOf(CharPool.SLASH);
541
542 x = key.indexOf(CharPool.SLASH, x + 1);
543
544 int y = key.lastIndexOf(CharPool.SLASH);
545
546 return key.substring(x + 1, y);
547 }
548
549 protected String getHeadVersionLabel(
550 long companyId, long repositoryId, String fileName)
551 throws PortalException, S3ServiceException {
552
553 S3Object[] s3Objects = _s3Service.listObjects(
554 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
555 null);
556
557 String[] keys = new String[s3Objects.length];
558
559 for (int i = 0; i < s3Objects.length; i++) {
560 S3Object s3Object = s3Objects[i];
561
562 keys[i] = s3Object.getKey();
563 }
564
565 if (keys.length > 0) {
566 Arrays.sort(keys);
567
568 String headKey = keys[keys.length - 1];
569
570 int x = headKey.lastIndexOf(CharPool.SLASH);
571
572 return headKey.substring(x + 1);
573 }
574 else {
575 throw new NoSuchFileException(fileName);
576 }
577 }
578
579 protected String getKey(long companyId, long repositoryId) {
580 StringBundler sb = new StringBundler(4);
581
582 sb.append(companyId);
583 sb.append(StringPool.SLASH);
584 sb.append(repositoryId);
585 sb.append(StringPool.SLASH);
586
587 return sb.toString();
588 }
589
590 protected String getKey(
591 long companyId, long repositoryId, String fileName) {
592
593 StringBundler sb = new StringBundler(6);
594
595 sb.append(companyId);
596 sb.append(StringPool.SLASH);
597 sb.append(repositoryId);
598 sb.append(StringPool.SLASH);
599 sb.append(fileName);
600 sb.append(StringPool.SLASH);
601
602 return sb.toString();
603 }
604
605 protected String getKey(
606 long companyId, long repositoryId, String fileName,
607 String versionLabel) {
608
609 StringBundler sb = new StringBundler(7);
610
611 sb.append(companyId);
612 sb.append(StringPool.SLASH);
613 sb.append(repositoryId);
614 sb.append(StringPool.SLASH);
615 sb.append(fileName);
616 sb.append(StringPool.SLASH);
617 sb.append(versionLabel);
618
619 return sb.toString();
620 }
621
622 protected S3Bucket getS3Bucket() throws S3ServiceException {
623 if (Validator.isNull(_BUCKET_NAME)) {
624 throw new S3ServiceException("S3 bucket name is not set");
625 }
626 else {
627 return getS3Service().getBucket(_BUCKET_NAME);
628 }
629 }
630
631 protected S3Service getS3Service() throws S3ServiceException {
632 AWSCredentials credentials = getAWSCredentials();
633
634 return new RestS3Service(credentials);
635 }
636
637 protected File getTempFile(S3Object s3Object, String fileName)
638 throws IOException, ServiceException {
639
640 StringBundler sb = new StringBundler(5);
641
642 sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
643 sb.append(_TEMP_DIR_NAME);
644 sb.append(
645 DateUtil.getCurrentDate(
646 _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
647 sb.append(fileName);
648
649 Date lastModifiedDate = s3Object.getLastModifiedDate();
650
651 sb.append(lastModifiedDate.getTime());
652
653 String tempFileName = sb.toString();
654
655 File tempFile = new File(tempFileName);
656
657 if (tempFile.exists() &&
658 (tempFile.lastModified() >= lastModifiedDate.getTime())) {
659
660 return tempFile;
661 }
662
663 InputStream inputStream = s3Object.getDataInputStream();
664
665 if (inputStream == null) {
666 throw new IOException("S3 object input stream is null");
667 }
668
669 OutputStream outputStream = null;
670
671 try {
672 File parentFile = tempFile.getParentFile();
673
674 parentFile.mkdirs();
675
676 outputStream = new FileOutputStream(tempFile);
677
678 StreamUtil.transfer(inputStream, outputStream);
679 }
680 finally {
681 StreamUtil.cleanUp(inputStream, outputStream);
682 }
683
684 return tempFile;
685 }
686
687 private static final String _ACCESS_KEY = PropsUtil.get(
688 PropsKeys.DL_STORE_S3_ACCESS_KEY);
689
690 private static final String _BUCKET_NAME = PropsUtil.get(
691 PropsKeys.DL_STORE_S3_BUCKET_NAME);
692
693 private static final String _SECRET_KEY = PropsUtil.get(
694 PropsKeys.DL_STORE_S3_SECRET_KEY);
695
696 private static final String _TEMP_DIR_NAME = "/liferay/s3";
697
698 private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
699
700 private static Log _log = LogFactoryUtil.getLog(S3Store.class);
701
702 private int _calledGetFileCount;
703 private S3Bucket _s3Bucket;
704 private S3Service _s3Service;
705
706 }