001
014
015 package com.liferay.portlet.documentlibrary.store;
016
017 import com.liferay.portal.kernel.exception.PortalException;
018 import com.liferay.portal.kernel.exception.SystemException;
019 import com.liferay.portal.kernel.log.Log;
020 import com.liferay.portal.kernel.log.LogFactoryUtil;
021 import com.liferay.portal.kernel.util.CharPool;
022 import com.liferay.portal.kernel.util.DateUtil;
023 import com.liferay.portal.kernel.util.FileUtil;
024 import com.liferay.portal.kernel.util.LocaleUtil;
025 import com.liferay.portal.kernel.util.PropsKeys;
026 import com.liferay.portal.kernel.util.StreamUtil;
027 import com.liferay.portal.kernel.util.StringBundler;
028 import com.liferay.portal.kernel.util.StringPool;
029 import com.liferay.portal.kernel.util.SystemProperties;
030 import com.liferay.portal.kernel.util.Time;
031 import com.liferay.portal.kernel.util.Validator;
032 import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033 import com.liferay.portal.util.PropsUtil;
034 import com.liferay.portal.util.PropsValues;
035 import com.liferay.portlet.documentlibrary.NoSuchFileException;
036
037 import java.io.File;
038 import java.io.FileInputStream;
039 import java.io.FileOutputStream;
040 import java.io.IOException;
041 import java.io.InputStream;
042 import java.io.OutputStream;
043
044 import java.util.ArrayList;
045 import java.util.Arrays;
046 import java.util.Date;
047 import java.util.List;
048
049 import org.jets3t.service.S3Service;
050 import org.jets3t.service.S3ServiceException;
051 import org.jets3t.service.ServiceException;
052 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
053 import org.jets3t.service.model.S3Bucket;
054 import org.jets3t.service.model.S3Object;
055 import org.jets3t.service.model.StorageObject;
056 import org.jets3t.service.security.AWSCredentials;
057
058
065 public class S3Store extends BaseStore {
066
067 public S3Store() {
068 try {
069 _s3Service = getS3Service();
070 _s3Bucket = getS3Bucket();
071 }
072 catch (S3ServiceException s3se) {
073 _log.error(s3se.getMessage());
074 }
075 }
076
077 @Override
078 public void addDirectory(
079 long companyId, long repositoryId, String dirName) {
080 }
081
082 @Override
083 public void addFile(
084 long companyId, long repositoryId, String fileName, InputStream is) {
085
086 try {
087 S3Object s3Object = new S3Object(
088 _s3Bucket,
089 getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
090
091 s3Object.setDataInputStream(is);
092
093 _s3Service.putObject(_s3Bucket, s3Object);
094 }
095 catch (S3ServiceException s3se) {
096 throw new SystemException(s3se);
097 }
098 finally {
099 StreamUtil.cleanUp(is);
100 }
101 }
102
103 @Override
104 public void checkRoot(long companyId) {
105 }
106
107 @Override
108 public void deleteDirectory(
109 long companyId, long repositoryId, String dirName) {
110
111 try {
112 S3Object[] s3Objects = _s3Service.listObjects(
113 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
114 null);
115
116 for (S3Object s3Object : s3Objects) {
117 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
118 }
119 }
120 catch (S3ServiceException s3se) {
121 throw new SystemException(s3se);
122 }
123 }
124
125 @Override
126 public void deleteFile(long companyId, long repositoryId, String fileName) {
127 try {
128 S3Object[] s3Objects = _s3Service.listObjects(
129 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
130 null);
131
132 for (S3Object s3Object : s3Objects) {
133 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
134 }
135 }
136 catch (S3ServiceException s3se) {
137 throw new SystemException(s3se);
138 }
139 }
140
141 @Override
142 public void deleteFile(
143 long companyId, long repositoryId, String fileName,
144 String versionLabel) {
145
146 try {
147 _s3Service.deleteObject(
148 _s3Bucket,
149 getKey(companyId, repositoryId, fileName, versionLabel));
150 }
151 catch (S3ServiceException s3se) {
152 throw new SystemException(s3se);
153 }
154 }
155
156 @Override
157 public File getFile(
158 long companyId, long repositoryId, String fileName,
159 String versionLabel)
160 throws PortalException {
161
162 try {
163 if (Validator.isNull(versionLabel)) {
164 versionLabel = getHeadVersionLabel(
165 companyId, repositoryId, fileName);
166 }
167
168 S3Object s3Object = _s3Service.getObject(
169 _s3Bucket.getName(),
170 getKey(companyId, repositoryId, fileName, versionLabel));
171
172 File tempFile = getTempFile(s3Object, fileName);
173
174 cleanUpTempFiles();
175
176 return tempFile;
177 }
178 catch (IOException ioe) {
179 throw new SystemException(ioe);
180 }
181 catch (ServiceException se) {
182 throw new SystemException(se);
183 }
184 }
185
186 @Override
187 public InputStream getFileAsStream(
188 long companyId, long repositoryId, String fileName,
189 String versionLabel)
190 throws PortalException {
191
192 try {
193 if (Validator.isNull(versionLabel)) {
194 versionLabel = getHeadVersionLabel(
195 companyId, repositoryId, fileName);
196 }
197
198 S3Object s3Object = _s3Service.getObject(
199 _s3Bucket.getName(),
200 getKey(companyId, repositoryId, fileName, versionLabel));
201
202 return s3Object.getDataInputStream();
203 }
204 catch (ServiceException se) {
205 throw new SystemException(se);
206 }
207 }
208
209 @Override
210 public String[] getFileNames(long companyId, long repositoryId) {
211 try {
212 S3Object[] s3Objects = _s3Service.listObjects(
213 _s3Bucket.getName(), getKey(companyId, repositoryId), null);
214
215 return getFileNames(s3Objects);
216 }
217 catch (S3ServiceException s3se) {
218 throw new SystemException(s3se);
219 }
220 }
221
222 @Override
223 public String[] getFileNames(
224 long companyId, long repositoryId, String dirName) {
225
226 try {
227 S3Object[] s3Objects = _s3Service.listObjects(
228 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
229 null);
230
231 return getFileNames(s3Objects);
232 }
233 catch (S3ServiceException s3se) {
234 throw new SystemException(s3se);
235 }
236 }
237
238 @Override
239 public long getFileSize(long companyId, long repositoryId, String fileName)
240 throws PortalException {
241
242 try {
243 String versionLabel = getHeadVersionLabel(
244 companyId, repositoryId, fileName);
245
246 StorageObject storageObject = _s3Service.getObjectDetails(
247 _s3Bucket.getName(),
248 getKey(companyId, repositoryId, fileName, versionLabel));
249
250 return storageObject.getContentLength();
251 }
252 catch (ServiceException se) {
253 throw new SystemException(se);
254 }
255 }
256
257 @Override
258 public boolean hasDirectory(
259 long companyId, long repositoryId, String dirName) {
260
261 return true;
262 }
263
264 @Override
265 public boolean hasFile(
266 long companyId, long repositoryId, String fileName,
267 String versionLabel) {
268
269 try {
270 S3Object[] s3Objects = _s3Service.listObjects(
271 _s3Bucket.getName(),
272 getKey(companyId, repositoryId, fileName, versionLabel), null);
273
274 if (s3Objects.length == 0) {
275 return false;
276 }
277 else {
278 return true;
279 }
280 }
281 catch (S3ServiceException s3se) {
282 throw new SystemException(s3se);
283 }
284 }
285
286 @Override
287 public void move(String srcDir, String destDir) {
288 }
289
290 @Override
291 public void updateFile(
292 long companyId, long repositoryId, long newRepositoryId,
293 String fileName) {
294
295 File tempFile = null;
296 InputStream is = null;
297 S3Object newS3Object = null;
298
299 try {
300 S3Object[] s3Objects = _s3Service.listObjects(
301 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
302 null);
303
304 for (S3Object oldS3Object : s3Objects) {
305 String oldKey = oldS3Object.getKey();
306
307 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
308
309 tempFile = new File(
310 SystemProperties.get(SystemProperties.TMP_DIR) +
311 File.separator + PortalUUIDUtil.generate());
312
313 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
314
315 is = new FileInputStream(tempFile);
316
317 String newPrefix = getKey(companyId, newRepositoryId);
318
319 int x = oldKey.indexOf(CharPool.SLASH);
320
321 x = oldKey.indexOf(CharPool.SLASH, x + 1);
322
323 String newKey = newPrefix + oldKey.substring(x);
324
325 newS3Object = new S3Object(_s3Bucket, newKey);
326
327 newS3Object.setDataInputStream(is);
328
329 _s3Service.putObject(_s3Bucket, newS3Object);
330 _s3Service.deleteObject(_s3Bucket, oldKey);
331 }
332 }
333 catch (IOException ioe) {
334 throw new SystemException(ioe);
335 }
336 catch (ServiceException se) {
337 throw new SystemException(se);
338 }
339 finally {
340 StreamUtil.cleanUp(is);
341
342 FileUtil.delete(tempFile);
343 }
344 }
345
346 @Override
347 public void updateFile(
348 long companyId, long repositoryId, String fileName,
349 String newFileName) {
350
351 File tempFile = null;
352 InputStream is = null;
353 S3Object newS3Object = null;
354
355 try {
356 S3Object[] s3Objects = _s3Service.listObjects(
357 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
358 null);
359
360 for (S3Object oldS3Object : s3Objects) {
361 String oldKey = oldS3Object.getKey();
362
363 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
364
365 tempFile = new File(
366 SystemProperties.get(SystemProperties.TMP_DIR) +
367 File.separator + PortalUUIDUtil.generate());
368
369 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
370
371 oldS3Object.closeDataInputStream();
372
373 is = new FileInputStream(tempFile);
374
375 String newPrefix = getKey(companyId, repositoryId, newFileName);
376
377 int x = oldKey.indexOf(StringPool.SLASH);
378
379 x = oldKey.indexOf(CharPool.SLASH, x + 1);
380 x = oldKey.indexOf(CharPool.SLASH, x + 1);
381
382 String newKey = newPrefix + oldKey.substring(x);
383
384 newS3Object = new S3Object(_s3Bucket, newKey);
385
386 newS3Object.setDataInputStream(is);
387
388 _s3Service.putObject(_s3Bucket, newS3Object);
389 _s3Service.deleteObject(_s3Bucket, oldKey);
390 }
391 }
392 catch (IOException ioe) {
393 throw new SystemException(ioe);
394 }
395 catch (ServiceException se) {
396 throw new SystemException(se);
397 }
398 finally {
399 StreamUtil.cleanUp(is);
400
401 FileUtil.delete(tempFile);
402 }
403 }
404
405 @Override
406 public void updateFile(
407 long companyId, long repositoryId, String fileName, String versionLabel,
408 InputStream is) {
409
410 try {
411 S3Object s3Object = new S3Object(
412 _s3Bucket,
413 getKey(companyId, repositoryId, fileName, versionLabel));
414
415 s3Object.setDataInputStream(is);
416
417 _s3Service.putObject(_s3Bucket, s3Object);
418 }
419 catch (S3ServiceException s3se) {
420 throw new SystemException(s3se);
421 }
422 finally {
423 StreamUtil.cleanUp(is);
424 }
425 }
426
427 protected void cleanUpTempFiles() {
428 _calledGetFileCount++;
429
430 if (_calledGetFileCount <
431 PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
432
433 return;
434 }
435
436 synchronized (this) {
437 if (_calledGetFileCount == 0) {
438 return;
439 }
440
441 _calledGetFileCount = 0;
442
443 String tempDirName =
444 SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
445
446 File tempDir = new File(tempDirName);
447
448 long lastModified = System.currentTimeMillis();
449
450 lastModified -=
451 (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
452
453 cleanUpTempFiles(tempDir, lastModified);
454 }
455 }
456
457 protected void cleanUpTempFiles(File file, long lastModified) {
458 if (!file.isDirectory()) {
459 return;
460 }
461
462 String[] fileNames = FileUtil.listDirs(file);
463
464 if (fileNames.length == 0) {
465 if (file.lastModified() < lastModified) {
466 FileUtil.deltree(file);
467
468 return;
469 }
470 }
471 else {
472 for (String fileName : fileNames) {
473 cleanUpTempFiles(new File(file, fileName), lastModified);
474 }
475
476 String[] subfileNames = file.list();
477
478 if (subfileNames.length == 0) {
479 FileUtil.deltree(file);
480
481 return;
482 }
483 }
484 }
485
486 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
487 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
488 throw new S3ServiceException(
489 "S3 access and secret keys are not set");
490 }
491 else {
492 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
493 }
494 }
495
496 protected String getFileName(String key) {
497
498
499
500
501 int x = key.indexOf(CharPool.SLASH);
502
503 x = key.indexOf(CharPool.SLASH, x + 1);
504
505 int y = key.lastIndexOf(CharPool.SLASH);
506
507 return key.substring(x, y);
508 }
509
510 protected String[] getFileNames(S3Object[] s3Objects) {
511 List<String> fileNames = new ArrayList<String>();
512
513 for (S3Object s3Object : s3Objects) {
514 String fileName = getFileName(s3Object.getKey());
515
516 fileNames.add(fileName);
517 }
518
519 return fileNames.toArray(new String[fileNames.size()]);
520 }
521
522 protected String getHeadVersionLabel(
523 long companyId, long repositoryId, String fileName)
524 throws PortalException, S3ServiceException {
525
526 S3Object[] s3Objects = _s3Service.listObjects(
527 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
528 null);
529
530 String[] keys = new String[s3Objects.length];
531
532 for (int i = 0; i < s3Objects.length; i++) {
533 S3Object s3Object = s3Objects[i];
534
535 keys[i] = s3Object.getKey();
536 }
537
538 if (keys.length > 0) {
539 Arrays.sort(keys);
540
541 String headKey = keys[keys.length - 1];
542
543 int x = headKey.lastIndexOf(CharPool.SLASH);
544
545 return headKey.substring(x + 1);
546 }
547 else {
548 throw new NoSuchFileException(fileName);
549 }
550 }
551
552 protected String getKey(long companyId, long repositoryId) {
553 return companyId + StringPool.SLASH + repositoryId;
554 }
555
556 protected String getKey(
557 long companyId, long repositoryId, String fileName) {
558
559 StringBundler sb = new StringBundler(4);
560
561 sb.append(companyId);
562 sb.append(StringPool.SLASH);
563 sb.append(repositoryId);
564 sb.append(getNormalizedFileName(fileName));
565
566 return sb.toString();
567 }
568
569 protected String getKey(
570 long companyId, long repositoryId, String fileName,
571 String versionLabel) {
572
573 StringBundler sb = new StringBundler(6);
574
575 sb.append(companyId);
576 sb.append(StringPool.SLASH);
577 sb.append(repositoryId);
578 sb.append(getNormalizedFileName(fileName));
579 sb.append(StringPool.SLASH);
580 sb.append(versionLabel);
581
582 return sb.toString();
583 }
584
585 protected String getNormalizedFileName(String fileName) {
586 String normalizedFileName = fileName;
587
588 if (!fileName.startsWith(StringPool.SLASH)) {
589 normalizedFileName = StringPool.SLASH + normalizedFileName;
590 }
591
592 if (fileName.endsWith(StringPool.SLASH)) {
593 normalizedFileName = normalizedFileName.substring(
594 0, normalizedFileName.length() - 1);
595 }
596
597 return normalizedFileName;
598 }
599
600 protected S3Bucket getS3Bucket() throws S3ServiceException {
601 if (Validator.isNull(_BUCKET_NAME)) {
602 throw new S3ServiceException("S3 bucket name is not set");
603 }
604 else {
605 return getS3Service().getBucket(_BUCKET_NAME);
606 }
607 }
608
609 protected S3Service getS3Service() throws S3ServiceException {
610 AWSCredentials credentials = getAWSCredentials();
611
612 return new RestS3Service(credentials);
613 }
614
615 protected File getTempFile(S3Object s3Object, String fileName)
616 throws IOException, ServiceException {
617
618 StringBundler sb = new StringBundler(5);
619
620 sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
621 sb.append(_TEMP_DIR_NAME);
622 sb.append(
623 DateUtil.getCurrentDate(
624 _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
625 sb.append(getNormalizedFileName(fileName));
626
627 Date lastModifiedDate = s3Object.getLastModifiedDate();
628
629 sb.append(lastModifiedDate.getTime());
630
631 String tempFileName = sb.toString();
632
633 File tempFile = new File(tempFileName);
634
635 if (tempFile.exists() &&
636 (tempFile.lastModified() >= lastModifiedDate.getTime())) {
637
638 return tempFile;
639 }
640
641 InputStream inputStream = s3Object.getDataInputStream();
642
643 if (inputStream == null) {
644 throw new IOException("S3 object input stream is null");
645 }
646
647 OutputStream outputStream = null;
648
649 try {
650 File parentFile = tempFile.getParentFile();
651
652 parentFile.mkdirs();
653
654 outputStream = new FileOutputStream(tempFile);
655
656 StreamUtil.transfer(inputStream, outputStream);
657 }
658 finally {
659 StreamUtil.cleanUp(inputStream, outputStream);
660 }
661
662 return tempFile;
663 }
664
665 private static final String _ACCESS_KEY = PropsUtil.get(
666 PropsKeys.DL_STORE_S3_ACCESS_KEY);
667
668 private static final String _BUCKET_NAME = PropsUtil.get(
669 PropsKeys.DL_STORE_S3_BUCKET_NAME);
670
671 private static final String _SECRET_KEY = PropsUtil.get(
672 PropsKeys.DL_STORE_S3_SECRET_KEY);
673
674 private static final String _TEMP_DIR_NAME = "/liferay/s3";
675
676 private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
677
678 private static Log _log = LogFactoryUtil.getLog(S3Store.class);
679
680 private int _calledGetFileCount;
681 private S3Bucket _s3Bucket;
682 private S3Service _s3Service;
683
684 }