001
014
015 package com.liferay.portlet.documentlibrary.store;
016
017 import com.liferay.portal.kernel.exception.PortalException;
018 import com.liferay.portal.kernel.exception.SystemException;
019 import com.liferay.portal.kernel.log.Log;
020 import com.liferay.portal.kernel.log.LogFactoryUtil;
021 import com.liferay.portal.kernel.util.CharPool;
022 import com.liferay.portal.kernel.util.DateUtil;
023 import com.liferay.portal.kernel.util.FileUtil;
024 import com.liferay.portal.kernel.util.LocaleUtil;
025 import com.liferay.portal.kernel.util.PropsKeys;
026 import com.liferay.portal.kernel.util.StreamUtil;
027 import com.liferay.portal.kernel.util.StringBundler;
028 import com.liferay.portal.kernel.util.StringPool;
029 import com.liferay.portal.kernel.util.SystemProperties;
030 import com.liferay.portal.kernel.util.Time;
031 import com.liferay.portal.kernel.util.Validator;
032 import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033 import com.liferay.portal.util.PropsUtil;
034 import com.liferay.portal.util.PropsValues;
035 import com.liferay.portlet.documentlibrary.NoSuchFileException;
036
037 import java.io.File;
038 import java.io.FileInputStream;
039 import java.io.FileOutputStream;
040 import java.io.IOException;
041 import java.io.InputStream;
042 import java.io.OutputStream;
043
044 import java.util.ArrayList;
045 import java.util.Arrays;
046 import java.util.Date;
047 import java.util.List;
048
049 import org.jets3t.service.S3Service;
050 import org.jets3t.service.S3ServiceException;
051 import org.jets3t.service.ServiceException;
052 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
053 import org.jets3t.service.model.S3Bucket;
054 import org.jets3t.service.model.S3Object;
055 import org.jets3t.service.model.StorageObject;
056 import org.jets3t.service.security.AWSCredentials;
057
058
065 public class S3Store extends BaseStore {
066
067 public S3Store() {
068 try {
069 _s3Service = getS3Service();
070 _s3Bucket = getS3Bucket();
071 }
072 catch (S3ServiceException s3se) {
073 _log.error(s3se.getMessage());
074 }
075 }
076
077 @Override
078 public void addDirectory(
079 long companyId, long repositoryId, String dirName) {
080 }
081
082 @Override
083 public void addFile(
084 long companyId, long repositoryId, String fileName, InputStream is)
085 throws SystemException {
086
087 try {
088 S3Object s3Object = new S3Object(
089 _s3Bucket,
090 getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
091
092 s3Object.setDataInputStream(is);
093
094 _s3Service.putObject(_s3Bucket, s3Object);
095
096 is.close();
097 }
098 catch (IOException ioe) {
099 throw new SystemException(ioe);
100 }
101 catch (S3ServiceException s3se) {
102 throw new SystemException(s3se);
103 }
104 }
105
106 @Override
107 public void checkRoot(long companyId) {
108 }
109
110 @Override
111 public void deleteDirectory(
112 long companyId, long repositoryId, String dirName)
113 throws SystemException {
114
115 try {
116 S3Object[] s3Objects = _s3Service.listObjects(
117 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
118 null);
119
120 for (S3Object s3Object : s3Objects) {
121 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
122 }
123 }
124 catch (S3ServiceException s3se) {
125 throw new SystemException(s3se);
126 }
127 }
128
129 @Override
130 public void deleteFile(long companyId, long repositoryId, String fileName)
131 throws SystemException {
132
133 try {
134 S3Object[] s3Objects = _s3Service.listObjects(
135 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
136 null);
137
138 for (S3Object s3Object : s3Objects) {
139 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
140 }
141 }
142 catch (S3ServiceException s3se) {
143 throw new SystemException(s3se);
144 }
145 }
146
147 @Override
148 public void deleteFile(
149 long companyId, long repositoryId, String fileName,
150 String versionLabel)
151 throws SystemException {
152
153 try {
154 _s3Service.deleteObject(
155 _s3Bucket,
156 getKey(companyId, repositoryId, fileName, versionLabel));
157 }
158 catch (S3ServiceException s3se) {
159 throw new SystemException(s3se);
160 }
161 }
162
163 @Override
164 public File getFile(
165 long companyId, long repositoryId, String fileName,
166 String versionLabel)
167 throws PortalException, SystemException {
168
169 try {
170 if (Validator.isNull(versionLabel)) {
171 versionLabel = getHeadVersionLabel(
172 companyId, repositoryId, fileName);
173 }
174
175 S3Object s3Object = _s3Service.getObject(
176 _s3Bucket.getName(),
177 getKey(companyId, repositoryId, fileName, versionLabel));
178
179 File tempFile = getTempFile(s3Object, fileName);
180
181 cleanUpTempFiles();
182
183 return tempFile;
184 }
185 catch (IOException ioe) {
186 throw new SystemException(ioe);
187 }
188 catch (ServiceException se) {
189 throw new SystemException(se);
190 }
191 }
192
193 @Override
194 public InputStream getFileAsStream(
195 long companyId, long repositoryId, String fileName,
196 String versionLabel)
197 throws PortalException, SystemException {
198
199 try {
200 if (Validator.isNull(versionLabel)) {
201 versionLabel = getHeadVersionLabel(
202 companyId, repositoryId, fileName);
203 }
204
205 S3Object s3Object = _s3Service.getObject(
206 _s3Bucket.getName(),
207 getKey(companyId, repositoryId, fileName, versionLabel));
208
209 return s3Object.getDataInputStream();
210 }
211 catch (ServiceException se) {
212 throw new SystemException(se);
213 }
214 }
215
216 public String[] getFileNames(long companyId, long repositoryId)
217 throws SystemException {
218
219 try {
220 S3Object[] s3Objects = _s3Service.listObjects(
221 _s3Bucket.getName(), getKey(companyId, repositoryId), null);
222
223 return getFileNames(s3Objects);
224 }
225 catch (S3ServiceException s3se) {
226 throw new SystemException(s3se);
227 }
228 }
229
230 @Override
231 public String[] getFileNames(
232 long companyId, long repositoryId, String dirName)
233 throws SystemException {
234
235 try {
236 S3Object[] s3Objects = _s3Service.listObjects(
237 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
238 null);
239
240 return getFileNames(s3Objects);
241 }
242 catch (S3ServiceException s3se) {
243 throw new SystemException(s3se);
244 }
245 }
246
247 @Override
248 public long getFileSize(long companyId, long repositoryId, String fileName)
249 throws PortalException, SystemException {
250
251 try {
252 String versionLabel = getHeadVersionLabel(
253 companyId, repositoryId, fileName);
254
255 StorageObject storageObject = _s3Service.getObjectDetails(
256 _s3Bucket.getName(),
257 getKey(companyId, repositoryId, fileName, versionLabel));
258
259 return storageObject.getContentLength();
260 }
261 catch (ServiceException se) {
262 throw new SystemException(se);
263 }
264 }
265
266 @Override
267 public boolean hasDirectory(
268 long companyId, long repositoryId, String dirName) {
269
270 return true;
271 }
272
273 @Override
274 public boolean hasFile(
275 long companyId, long repositoryId, String fileName,
276 String versionLabel)
277 throws SystemException {
278
279 try {
280 S3Object[] s3Objects = _s3Service.listObjects(
281 _s3Bucket.getName(),
282 getKey(companyId, repositoryId, fileName, versionLabel), null);
283
284 if (s3Objects.length == 0) {
285 return false;
286 }
287 else {
288 return true;
289 }
290 }
291 catch (S3ServiceException s3se) {
292 throw new SystemException(s3se);
293 }
294 }
295
296 @Override
297 public void move(String srcDir, String destDir) {
298 }
299
300 @Override
301 public void updateFile(
302 long companyId, long repositoryId, long newRepositoryId,
303 String fileName)
304 throws SystemException {
305
306 try {
307 S3Object[] s3Objects = _s3Service.listObjects(
308 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
309 null);
310
311 for (S3Object oldS3Object : s3Objects) {
312 String oldKey = oldS3Object.getKey();
313
314 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
315
316 File tempFile = new File(
317 SystemProperties.get(SystemProperties.TMP_DIR) +
318 File.separator + PortalUUIDUtil.generate());
319
320 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
321
322 InputStream is = new FileInputStream(tempFile);
323
324 String newPrefix = getKey(companyId, newRepositoryId);
325
326 int x = oldKey.indexOf(CharPool.SLASH);
327
328 x = oldKey.indexOf(CharPool.SLASH, x + 1);
329
330 String newKey = newPrefix + oldKey.substring(x + 1);
331
332 S3Object newS3Object = new S3Object(_s3Bucket, newKey);
333
334 newS3Object.setDataInputStream(is);
335
336 _s3Service.putObject(_s3Bucket, newS3Object);
337 _s3Service.deleteObject(_s3Bucket, oldKey);
338
339 is.close();
340
341 FileUtil.delete(tempFile);
342 }
343 }
344 catch (IOException ioe) {
345 throw new SystemException(ioe);
346 }
347 catch (ServiceException se) {
348 throw new SystemException(se);
349 }
350 }
351
352 public void updateFile(
353 long companyId, long repositoryId, String fileName,
354 String newFileName)
355 throws SystemException {
356
357 try {
358 S3Object[] s3Objects = _s3Service.listObjects(
359 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
360 null);
361
362 for (S3Object oldS3Object : s3Objects) {
363 String oldKey = oldS3Object.getKey();
364
365 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
366
367 File tempFile = new File(
368 SystemProperties.get(SystemProperties.TMP_DIR) +
369 File.separator + PortalUUIDUtil.generate());
370
371 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
372
373 oldS3Object.closeDataInputStream();
374
375 InputStream is = new FileInputStream(tempFile);
376
377 String newPrefix = getKey(companyId, repositoryId, newFileName);
378
379 int x = oldKey.indexOf(StringPool.SLASH);
380
381 x = oldKey.indexOf(CharPool.SLASH, x + 1);
382 x = oldKey.indexOf(CharPool.SLASH, x + 1);
383
384 String newKey = newPrefix + oldKey.substring(x + 1);
385
386 S3Object newS3Object = new S3Object(_s3Bucket, newKey);
387
388 newS3Object.setDataInputStream(is);
389
390 _s3Service.putObject(_s3Bucket, newS3Object);
391 _s3Service.deleteObject(_s3Bucket, oldKey);
392
393 newS3Object.closeDataInputStream();
394
395 is.close();
396
397 FileUtil.delete(tempFile);
398 }
399 }
400 catch (IOException ioe) {
401 throw new SystemException(ioe);
402 }
403 catch (ServiceException se) {
404 throw new SystemException(se);
405 }
406 }
407
408 @Override
409 public void updateFile(
410 long companyId, long repositoryId, String fileName,
411 String versionLabel, InputStream is)
412 throws SystemException {
413
414 try {
415 S3Object s3Object = new S3Object(
416 _s3Bucket,
417 getKey(companyId, repositoryId, fileName, versionLabel));
418
419 s3Object.setDataInputStream(is);
420
421 _s3Service.putObject(_s3Bucket, s3Object);
422
423 is.close();
424 }
425 catch (IOException ioe) {
426 throw new SystemException(ioe);
427 }
428 catch (S3ServiceException s3se) {
429 throw new SystemException(s3se);
430 }
431 }
432
433 protected void cleanUpTempFiles() {
434 _calledGetFileCount++;
435
436 if (_calledGetFileCount <
437 PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
438
439 return;
440 }
441
442 synchronized (this) {
443 if (_calledGetFileCount == 0) {
444 return;
445 }
446
447 _calledGetFileCount = 0;
448
449 String tempDirName =
450 SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
451
452 File tempDir = new File(tempDirName);
453
454 long lastModified = System.currentTimeMillis();
455
456 lastModified -=
457 (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
458
459 cleanUpTempFiles(tempDir, lastModified);
460 }
461 }
462
463 protected void cleanUpTempFiles(File file, long lastModified) {
464 if (!file.isDirectory()) {
465 return;
466 }
467
468 String[] fileNames = FileUtil.listDirs(file);
469
470 if (fileNames.length == 0) {
471 if (file.lastModified() < lastModified) {
472 FileUtil.deltree(file);
473
474 return;
475 }
476 }
477 else {
478 for (String fileName : fileNames) {
479 cleanUpTempFiles(new File(file, fileName), lastModified);
480 }
481
482 String[] subfileNames = file.list();
483
484 if (subfileNames.length == 0) {
485 FileUtil.deltree(file);
486
487 return;
488 }
489 }
490 }
491
492 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
493 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
494 throw new S3ServiceException(
495 "S3 access and secret keys are not set");
496 }
497 else {
498 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
499 }
500 }
501
502 protected String getFileName(String key) {
503
504
505
506
507 int x = key.indexOf(CharPool.SLASH);
508
509 x = key.indexOf(CharPool.SLASH, x + 1);
510
511 int y = key.lastIndexOf(CharPool.SLASH);
512
513 return key.substring(x, y);
514 }
515
516 protected String[] getFileNames(S3Object[] s3Objects) {
517 List<String> fileNames = new ArrayList<String>();
518
519 for (S3Object s3Object : s3Objects) {
520 String fileName = getFileName(s3Object.getKey());
521
522 fileNames.add(fileName);
523 }
524
525 return fileNames.toArray(new String[fileNames.size()]);
526 }
527
528 protected String getHeadVersionLabel(
529 long companyId, long repositoryId, String fileName)
530 throws PortalException, S3ServiceException {
531
532 S3Object[] s3Objects = _s3Service.listObjects(
533 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
534 null);
535
536 String[] keys = new String[s3Objects.length];
537
538 for (int i = 0; i < s3Objects.length; i++) {
539 S3Object s3Object = s3Objects[i];
540
541 keys[i] = s3Object.getKey();
542 }
543
544 if (keys.length > 0) {
545 Arrays.sort(keys);
546
547 String headKey = keys[keys.length - 1];
548
549 int x = headKey.lastIndexOf(CharPool.SLASH);
550
551 return headKey.substring(x + 1);
552 }
553 else {
554 throw new NoSuchFileException(fileName);
555 }
556 }
557
558 protected String getKey(long companyId, long repositoryId) {
559 StringBundler sb = new StringBundler(4);
560
561 sb.append(companyId);
562 sb.append(StringPool.SLASH);
563 sb.append(repositoryId);
564
565 return sb.toString();
566 }
567
568 protected String getKey(
569 long companyId, long repositoryId, String fileName) {
570
571 StringBundler sb = new StringBundler(4);
572
573 sb.append(companyId);
574 sb.append(StringPool.SLASH);
575 sb.append(repositoryId);
576 sb.append(getNormalizedFileName(fileName));
577
578 return sb.toString();
579 }
580
581 protected String getKey(
582 long companyId, long repositoryId, String fileName,
583 String versionLabel) {
584
585 StringBundler sb = new StringBundler(6);
586
587 sb.append(companyId);
588 sb.append(StringPool.SLASH);
589 sb.append(repositoryId);
590 sb.append(getNormalizedFileName(fileName));
591 sb.append(StringPool.SLASH);
592 sb.append(versionLabel);
593
594 return sb.toString();
595 }
596
597 protected String getNormalizedFileName(String fileName) {
598 String normalizedFileName = fileName;
599
600 if (!fileName.startsWith(StringPool.SLASH)) {
601 normalizedFileName = StringPool.SLASH + normalizedFileName;
602 }
603
604 if (fileName.endsWith(StringPool.SLASH)) {
605 normalizedFileName = normalizedFileName.substring(
606 0, normalizedFileName.length() - 1);
607 }
608
609 return normalizedFileName;
610 }
611
612 protected S3Bucket getS3Bucket() throws S3ServiceException {
613 if (Validator.isNull(_BUCKET_NAME)) {
614 throw new S3ServiceException("S3 bucket name is not set");
615 }
616 else {
617 return getS3Service().getBucket(_BUCKET_NAME);
618 }
619 }
620
621 protected S3Service getS3Service() throws S3ServiceException {
622 AWSCredentials credentials = getAWSCredentials();
623
624 return new RestS3Service(credentials);
625 }
626
627 protected File getTempFile(S3Object s3Object, String fileName)
628 throws IOException, ServiceException {
629
630 StringBundler sb = new StringBundler(5);
631
632 sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
633 sb.append(_TEMP_DIR_NAME);
634 sb.append(
635 DateUtil.getCurrentDate(
636 _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
637 sb.append(getNormalizedFileName(fileName));
638
639 Date lastModifiedDate = s3Object.getLastModifiedDate();
640
641 sb.append(lastModifiedDate.getTime());
642
643 String tempFileName = sb.toString();
644
645 File tempFile = new File(tempFileName);
646
647 if (tempFile.exists() &&
648 (tempFile.lastModified() >= lastModifiedDate.getTime())) {
649
650 return tempFile;
651 }
652
653 InputStream inputStream = s3Object.getDataInputStream();
654
655 if (inputStream == null) {
656 throw new IOException("S3 object input stream is null");
657 }
658
659 OutputStream outputStream = null;
660
661 try {
662 File parentFile = tempFile.getParentFile();
663
664 parentFile.mkdirs();
665
666 outputStream = new FileOutputStream(tempFile);
667
668 StreamUtil.transfer(inputStream, outputStream);
669 }
670 finally {
671 StreamUtil.cleanUp(inputStream, outputStream);
672 }
673
674 return tempFile;
675 }
676
677 private static final String _ACCESS_KEY = PropsUtil.get(
678 PropsKeys.DL_STORE_S3_ACCESS_KEY);
679
680 private static final String _BUCKET_NAME = PropsUtil.get(
681 PropsKeys.DL_STORE_S3_BUCKET_NAME);
682
683 private static final String _SECRET_KEY = PropsUtil.get(
684 PropsKeys.DL_STORE_S3_SECRET_KEY);
685
686 private static final String _TEMP_DIR_NAME = "/liferay/s3";
687
688 private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
689
690 private static Log _log = LogFactoryUtil.getLog(S3Store.class);
691
692 private int _calledGetFileCount;
693 private S3Bucket _s3Bucket;
694 private S3Service _s3Service;
695
696 }