001    /**
002     * Copyright (c) 2000-2013 Liferay, Inc. All rights reserved.
003     *
004     * The contents of this file are subject to the terms of the Liferay Enterprise
005     * Subscription License ("License"). You may not use this file except in
006     * compliance with the License. You can obtain a copy of the License by
007     * contacting Liferay, Inc. See the License for the specific language governing
008     * permissions and limitations under the License, including but not limited to
009     * distribution rights of the Software.
010     *
011     *
012     *
013     */
014    
015    package com.liferay.portlet.documentlibrary.store;
016    
017    import com.liferay.portal.kernel.exception.PortalException;
018    import com.liferay.portal.kernel.exception.SystemException;
019    import com.liferay.portal.kernel.log.Log;
020    import com.liferay.portal.kernel.log.LogFactoryUtil;
021    import com.liferay.portal.kernel.util.CharPool;
022    import com.liferay.portal.kernel.util.DateUtil;
023    import com.liferay.portal.kernel.util.FileUtil;
024    import com.liferay.portal.kernel.util.GetterUtil;
025    import com.liferay.portal.kernel.util.LocaleUtil;
026    import com.liferay.portal.kernel.util.PropsKeys;
027    import com.liferay.portal.kernel.util.StreamUtil;
028    import com.liferay.portal.kernel.util.StringBundler;
029    import com.liferay.portal.kernel.util.StringPool;
030    import com.liferay.portal.kernel.util.SystemProperties;
031    import com.liferay.portal.kernel.util.Time;
032    import com.liferay.portal.kernel.util.Validator;
033    import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
034    import com.liferay.portal.util.PropsUtil;
035    import com.liferay.portal.util.PropsValues;
036    import com.liferay.portlet.documentlibrary.NoSuchFileException;
037    
038    import java.io.File;
039    import java.io.FileInputStream;
040    import java.io.FileOutputStream;
041    import java.io.IOException;
042    import java.io.InputStream;
043    import java.io.OutputStream;
044    
045    import java.util.ArrayList;
046    import java.util.Arrays;
047    import java.util.Date;
048    import java.util.List;
049    
050    import org.jets3t.service.S3Service;
051    import org.jets3t.service.S3ServiceException;
052    import org.jets3t.service.ServiceException;
053    import org.jets3t.service.impl.rest.httpclient.RestS3Service;
054    import org.jets3t.service.model.S3Bucket;
055    import org.jets3t.service.model.S3Object;
056    import org.jets3t.service.model.StorageObject;
057    import org.jets3t.service.security.AWSCredentials;
058    import org.jets3t.service.utils.MultipartUtils;
059    
060    /**
061     * @author Brian Wing Shun Chan
062     * @author Sten Martinez
063     * @author Edward Han
064     * @author Vilmos Papp
065     * @author Mate Thurzo
066     */
067    public class S3Store extends BaseStore {
068    
069            public S3Store() {
070                    try {
071                            _s3Service = getS3Service();
072                            _s3Bucket = getS3Bucket();
073                    }
074                    catch (S3ServiceException s3se) {
075                            _log.error(s3se.getMessage());
076                    }
077            }
078    
079            @Override
080            public void addDirectory(
081                    long companyId, long repositoryId, String dirName) {
082            }
083    
084            @Override
085            public void addFile(
086                            long companyId, long repositoryId, String fileName, InputStream is)
087                    throws SystemException {
088    
089                    try {
090                            File file = FileUtil.createTempFile(is);
091    
092                            addFile(companyId, repositoryId, fileName, file);
093                    }
094                    catch (IOException ioe) {
095                            throw new SystemException(ioe);
096                    }
097                    finally {
098                            StreamUtil.cleanUp(is);
099                    }
100            }
101    
102            @Override
103            public void addFile(
104                            long companyId, long repositoryId, String fileName, File file)
105                    throws SystemException {
106    
107                    putObject(
108                            getKey(companyId, repositoryId, fileName, VERSION_DEFAULT), file);
109            }
110    
111            @Override
112            public void checkRoot(long companyId) {
113            }
114    
115            @Override
116            public void deleteDirectory(
117                            long companyId, long repositoryId, String dirName)
118                    throws SystemException {
119    
120                    try {
121                            S3Object[] s3Objects = _s3Service.listObjects(
122                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
123                                    null);
124    
125                            for (S3Object s3Object : s3Objects) {
126                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
127                            }
128                    }
129                    catch (S3ServiceException s3se) {
130                            throw new SystemException(s3se);
131                    }
132            }
133    
134            @Override
135            public void deleteFile(long companyId, long repositoryId, String fileName)
136                    throws SystemException {
137    
138                    try {
139                            S3Object[] s3Objects = _s3Service.listObjects(
140                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
141                                    null);
142    
143                            for (S3Object s3Object : s3Objects) {
144                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
145                            }
146                    }
147                    catch (S3ServiceException s3se) {
148                            throw new SystemException(s3se);
149                    }
150            }
151    
152            @Override
153            public void deleteFile(
154                            long companyId, long repositoryId, String fileName,
155                            String versionLabel)
156                    throws SystemException {
157    
158                    try {
159                            _s3Service.deleteObject(
160                                    _s3Bucket,
161                                    getKey(companyId, repositoryId, fileName, versionLabel));
162                    }
163                    catch (S3ServiceException s3se) {
164                            throw new SystemException(s3se);
165                    }
166            }
167    
168            @Override
169            public File getFile(
170                            long companyId, long repositoryId, String fileName,
171                            String versionLabel)
172                    throws PortalException, SystemException {
173    
174                    try {
175                            if (Validator.isNull(versionLabel)) {
176                                    versionLabel = getHeadVersionLabel(
177                                            companyId, repositoryId, fileName);
178                            }
179    
180                            S3Object s3Object = _s3Service.getObject(
181                                    _s3Bucket.getName(),
182                                    getKey(companyId, repositoryId, fileName, versionLabel));
183    
184                            File tempFile = getTempFile(s3Object, fileName);
185    
186                            cleanUpTempFiles();
187    
188                            return tempFile;
189                    }
190                    catch (IOException ioe) {
191                            throw new SystemException(ioe);
192                    }
193                    catch (ServiceException se) {
194                            throw new SystemException(se);
195                    }
196            }
197    
198            @Override
199            public InputStream getFileAsStream(
200                            long companyId, long repositoryId, String fileName,
201                            String versionLabel)
202                    throws PortalException, SystemException {
203    
204                    try {
205                            if (Validator.isNull(versionLabel)) {
206                                    versionLabel = getHeadVersionLabel(
207                                            companyId, repositoryId, fileName);
208                            }
209    
210                            S3Object s3Object = _s3Service.getObject(
211                                    _s3Bucket.getName(),
212                                    getKey(companyId, repositoryId, fileName, versionLabel));
213    
214                            return s3Object.getDataInputStream();
215                    }
216                    catch (ServiceException se) {
217                            throw new SystemException(se);
218                    }
219            }
220    
221            @Override
222            public String[] getFileNames(long companyId, long repositoryId)
223                    throws SystemException {
224    
225                    try {
226                            S3Object[] s3Objects = _s3Service.listObjects(
227                                    _s3Bucket.getName(), getKey(companyId, repositoryId), null);
228    
229                            return getFileNames(s3Objects);
230                    }
231                    catch (S3ServiceException s3se) {
232                            throw new SystemException(s3se);
233                    }
234            }
235    
236            @Override
237            public String[] getFileNames(
238                            long companyId, long repositoryId, String dirName)
239                    throws SystemException {
240    
241                    try {
242                            S3Object[] s3Objects = _s3Service.listObjects(
243                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
244                                    null);
245    
246                            return getFileNames(s3Objects);
247                    }
248                    catch (S3ServiceException s3se) {
249                            throw new SystemException(s3se);
250                    }
251            }
252    
253            @Override
254            public long getFileSize(long companyId, long repositoryId, String fileName)
255                    throws PortalException, SystemException {
256    
257                    try {
258                            String versionLabel = getHeadVersionLabel(
259                                    companyId, repositoryId, fileName);
260    
261                            StorageObject storageObject = _s3Service.getObjectDetails(
262                                    _s3Bucket.getName(),
263                                    getKey(companyId, repositoryId, fileName, versionLabel));
264    
265                            return storageObject.getContentLength();
266                    }
267                    catch (ServiceException se) {
268                            throw new SystemException(se);
269                    }
270            }
271    
272            @Override
273            public boolean hasDirectory(
274                    long companyId, long repositoryId, String dirName) {
275    
276                    return true;
277            }
278    
279            @Override
280            public boolean hasFile(
281                            long companyId, long repositoryId, String fileName,
282                            String versionLabel)
283                    throws SystemException {
284    
285                    try {
286                            S3Object[] s3Objects = _s3Service.listObjects(
287                                    _s3Bucket.getName(),
288                                    getKey(companyId, repositoryId, fileName, versionLabel), null);
289    
290                            if (s3Objects.length == 0) {
291                                    return false;
292                            }
293                            else {
294                                    return true;
295                            }
296                    }
297                    catch (S3ServiceException s3se) {
298                            throw new SystemException(s3se);
299                    }
300            }
301    
302            @Override
303            public void move(String srcDir, String destDir) {
304            }
305    
306            @Override
307            public void updateFile(
308                            long companyId, long repositoryId, long newRepositoryId,
309                            String fileName)
310                    throws SystemException {
311    
312                    File tempFile = null;
313                    InputStream is = null;
314                    S3Object newS3Object = null;
315    
316                    try {
317                            S3Object[] s3Objects = _s3Service.listObjects(
318                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
319                                    null);
320    
321                            for (S3Object oldS3Object : s3Objects) {
322                                    String oldKey = oldS3Object.getKey();
323    
324                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
325    
326                                    tempFile = new File(
327                                            SystemProperties.get(SystemProperties.TMP_DIR) +
328                                                    File.separator + PortalUUIDUtil.generate());
329    
330                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
331    
332                                    is = new FileInputStream(tempFile);
333    
334                                    String newPrefix = getKey(companyId, newRepositoryId);
335    
336                                    int x = oldKey.indexOf(CharPool.SLASH);
337    
338                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
339    
340                                    String newKey = newPrefix + oldKey.substring(x);
341    
342                                    putObject(newKey, tempFile);
343    
344                                    _s3Service.deleteObject(_s3Bucket, oldKey);
345                            }
346                    }
347                    catch (IOException ioe) {
348                            throw new SystemException(ioe);
349                    }
350                    catch (ServiceException se) {
351                            throw new SystemException(se);
352                    }
353                    finally {
354                            StreamUtil.cleanUp(is);
355    
356                            FileUtil.delete(tempFile);
357                    }
358            }
359    
360            @Override
361            public void updateFile(
362                            long companyId, long repositoryId, String fileName,
363                            String newFileName)
364                    throws SystemException {
365    
366                    File tempFile = null;
367                    InputStream is = null;
368                    S3Object newS3Object = null;
369    
370                    try {
371                            S3Object[] s3Objects = _s3Service.listObjects(
372                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
373                                    null);
374    
375                            for (S3Object oldS3Object : s3Objects) {
376                                    String oldKey = oldS3Object.getKey();
377    
378                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
379    
380                                    tempFile = new File(
381                                            SystemProperties.get(SystemProperties.TMP_DIR) +
382                                                    File.separator + PortalUUIDUtil.generate());
383    
384                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
385    
386                                    oldS3Object.closeDataInputStream();
387    
388                                    is = new FileInputStream(tempFile);
389    
390                                    String newPrefix = getKey(companyId, repositoryId, newFileName);
391    
392                                    int x = oldKey.indexOf(StringPool.SLASH);
393    
394                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
395                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
396    
397                                    String newKey = newPrefix + oldKey.substring(x);
398    
399                                    putObject(newKey, tempFile);
400    
401                                    _s3Service.deleteObject(_s3Bucket, oldKey);
402                            }
403                    }
404                    catch (IOException ioe) {
405                            throw new SystemException(ioe);
406                    }
407                    catch (ServiceException se) {
408                            throw new SystemException(se);
409                    }
410                    finally {
411                            StreamUtil.cleanUp(is);
412    
413                            FileUtil.delete(tempFile);
414                    }
415            }
416    
417            @Override
418            public void updateFile(
419                            long companyId, long repositoryId, String fileName,
420                            String versionLabel, InputStream is)
421                    throws SystemException {
422    
423                    try {
424                            File file = FileUtil.createTempFile(is);
425    
426                            updateFile(companyId, repositoryId, fileName, versionLabel, file);
427                    }
428                    catch (Exception e) {
429                            throw new SystemException(e);
430                    }
431                    finally {
432                            StreamUtil.cleanUp(is);
433                    }
434            }
435    
436            @Override
437            public void updateFile(
438                            long companyId, long repositoryId, String fileName,
439                            String versionLabel, File file)
440                    throws PortalException, SystemException {
441    
442                    putObject(
443                            getKey(companyId, repositoryId, fileName, versionLabel), file);
444            }
445    
446            protected void cleanUpTempFiles() {
447                    _calledGetFileCount++;
448    
449                    if (_calledGetFileCount <
450                                    PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
451    
452                            return;
453                    }
454    
455                    synchronized (this) {
456                            if (_calledGetFileCount == 0) {
457                                    return;
458                            }
459    
460                            _calledGetFileCount = 0;
461    
462                            String tempDirName =
463                                    SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
464    
465                            File tempDir = new File(tempDirName);
466    
467                            long lastModified = System.currentTimeMillis();
468    
469                            lastModified -=
470                                    (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
471    
472                            cleanUpTempFiles(tempDir, lastModified);
473                    }
474            }
475    
476            protected void cleanUpTempFiles(File file, long lastModified) {
477                    if (!file.isDirectory()) {
478                            return;
479                    }
480    
481                    String[] fileNames = FileUtil.listDirs(file);
482    
483                    if (fileNames.length == 0) {
484                            if (file.lastModified() < lastModified) {
485                                    FileUtil.deltree(file);
486    
487                                    return;
488                            }
489                    }
490                    else {
491                            for (String fileName : fileNames) {
492                                    cleanUpTempFiles(new File(file, fileName), lastModified);
493                            }
494    
495                            String[] subfileNames = file.list();
496    
497                            if (subfileNames.length == 0) {
498                                    FileUtil.deltree(file);
499    
500                                    return;
501                            }
502                    }
503            }
504    
505            protected AWSCredentials getAWSCredentials() throws S3ServiceException {
506                    if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
507                            throw new S3ServiceException(
508                                    "S3 access and secret keys are not set");
509                    }
510                    else {
511                            return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
512                    }
513            }
514    
515            protected String getFileName(String key) {
516    
517                    // Convert /${companyId}/${repositoryId}/${dirName}/${fileName}
518                    // /${versionLabel} to /${dirName}/${fileName}
519    
520                    int x = key.indexOf(CharPool.SLASH);
521    
522                    x = key.indexOf(CharPool.SLASH, x + 1);
523    
524                    int y = key.lastIndexOf(CharPool.SLASH);
525    
526                    return key.substring(x, y);
527            }
528    
529            protected String[] getFileNames(S3Object[] s3Objects) {
530                    List<String> fileNames = new ArrayList<String>();
531    
532                    for (S3Object s3Object : s3Objects) {
533                            String fileName = getFileName(s3Object.getKey());
534    
535                            fileNames.add(fileName);
536                    }
537    
538                    return fileNames.toArray(new String[fileNames.size()]);
539            }
540    
541            protected String getHeadVersionLabel(
542                            long companyId, long repositoryId, String fileName)
543                    throws PortalException, S3ServiceException {
544    
545                    S3Object[] s3Objects = _s3Service.listObjects(
546                            _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
547                            null);
548    
549                    String[] keys = new String[s3Objects.length];
550    
551                    for (int i = 0; i < s3Objects.length; i++) {
552                            S3Object s3Object = s3Objects[i];
553    
554                            keys[i] = s3Object.getKey();
555                    }
556    
557                    if (keys.length > 0) {
558                            Arrays.sort(keys);
559    
560                            String headKey = keys[keys.length - 1];
561    
562                            int x = headKey.lastIndexOf(CharPool.SLASH);
563    
564                            return headKey.substring(x + 1);
565                    }
566                    else {
567                            throw new NoSuchFileException(fileName);
568                    }
569            }
570    
571            protected String getKey(long companyId, long repositoryId) {
572                    StringBundler sb = new StringBundler(4);
573    
574                    sb.append(companyId);
575                    sb.append(StringPool.SLASH);
576                    sb.append(repositoryId);
577    
578                    return sb.toString();
579            }
580    
581            protected String getKey(
582                    long companyId, long repositoryId, String fileName) {
583    
584                    StringBundler sb = new StringBundler(4);
585    
586                    sb.append(companyId);
587                    sb.append(StringPool.SLASH);
588                    sb.append(repositoryId);
589                    sb.append(getNormalizedFileName(fileName));
590    
591                    return sb.toString();
592            }
593    
594            protected String getKey(
595                    long companyId, long repositoryId, String fileName,
596                    String versionLabel) {
597    
598                    StringBundler sb = new StringBundler(6);
599    
600                    sb.append(companyId);
601                    sb.append(StringPool.SLASH);
602                    sb.append(repositoryId);
603                    sb.append(getNormalizedFileName(fileName));
604                    sb.append(StringPool.SLASH);
605                    sb.append(versionLabel);
606    
607                    return sb.toString();
608            }
609    
610            protected String getNormalizedFileName(String fileName) {
611                    String normalizedFileName = fileName;
612    
613                    if (!fileName.startsWith(StringPool.SLASH)) {
614                            normalizedFileName = StringPool.SLASH + normalizedFileName;
615                    }
616    
617                    if (fileName.endsWith(StringPool.SLASH)) {
618                            normalizedFileName = normalizedFileName.substring(
619                                    0, normalizedFileName.length() - 1);
620                    }
621    
622                    return normalizedFileName;
623            }
624    
625            protected S3Bucket getS3Bucket() throws S3ServiceException {
626                    if (Validator.isNull(_BUCKET_NAME)) {
627                            throw new S3ServiceException("S3 bucket name is not set");
628                    }
629                    else {
630                            return getS3Service().getBucket(_BUCKET_NAME);
631                    }
632            }
633    
634            protected S3Service getS3Service() throws S3ServiceException {
635                    AWSCredentials credentials = getAWSCredentials();
636    
637                    return new RestS3Service(credentials);
638            }
639    
640            protected File getTempFile(S3Object s3Object, String fileName)
641                    throws IOException, ServiceException {
642    
643                    StringBundler sb = new StringBundler(5);
644    
645                    sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
646                    sb.append(_TEMP_DIR_NAME);
647                    sb.append(
648                            DateUtil.getCurrentDate(
649                                    _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
650                    sb.append(getNormalizedFileName(fileName));
651    
652                    Date lastModifiedDate = s3Object.getLastModifiedDate();
653    
654                    sb.append(lastModifiedDate.getTime());
655    
656                    String tempFileName = sb.toString();
657    
658                    File tempFile = new File(tempFileName);
659    
660                    InputStream inputStream = s3Object.getDataInputStream();
661    
662                    if (tempFile.exists() &&
663                            (tempFile.lastModified() >= lastModifiedDate.getTime())) {
664    
665                            StreamUtil.cleanUp(inputStream);
666    
667                            return tempFile;
668                    }
669    
670                    if (inputStream == null) {
671                            throw new IOException("S3 object input stream is null");
672                    }
673    
674                    OutputStream outputStream = null;
675    
676                    try {
677                            File parentFile = tempFile.getParentFile();
678    
679                            FileUtil.mkdirs(parentFile);
680    
681                            outputStream = new FileOutputStream(tempFile);
682    
683                            StreamUtil.transfer(inputStream, outputStream);
684                    }
685                    finally {
686                            StreamUtil.cleanUp(inputStream, outputStream);
687                    }
688    
689                    return tempFile;
690            }
691    
692            protected void putObject(String key, File file) throws SystemException {
693                    try {
694                            MultipartUtils multipartUtils = new MultipartUtils(_PART_SIZE);
695    
696                            List<StorageObject> s3Objects = new ArrayList<StorageObject>();
697    
698                            S3Object s3Object = new S3Object(file);
699    
700                            s3Object.setBucketName(_BUCKET_NAME);
701                            s3Object.setKey(key);
702    
703                            s3Objects.add(s3Object);
704    
705                            multipartUtils.uploadObjects(
706                                    _BUCKET_NAME, _s3Service, s3Objects, null);
707                    }
708                    catch (Exception e) {
709                            throw new SystemException(e);
710                    }
711            }
712    
713            private static final String _ACCESS_KEY = PropsUtil.get(
714                    PropsKeys.DL_STORE_S3_ACCESS_KEY);
715    
716            private static final String _BUCKET_NAME = PropsUtil.get(
717                    PropsKeys.DL_STORE_S3_BUCKET_NAME);
718    
719            private static final long _PART_SIZE = GetterUtil.getLong(
720                    PropsUtil.get(PropsKeys.DL_STORE_S3_PART_SIZE));
721    
722            private static final String _SECRET_KEY = PropsUtil.get(
723                    PropsKeys.DL_STORE_S3_SECRET_KEY);
724    
725            private static final String _TEMP_DIR_NAME = "/liferay/s3";
726    
727            private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
728    
729            private static Log _log = LogFactoryUtil.getLog(S3Store.class);
730    
731            private int _calledGetFileCount;
732            private S3Bucket _s3Bucket;
733            private S3Service _s3Service;
734    
735    }