001    /**
002     * Copyright (c) 2000-2013 Liferay, Inc. All rights reserved.
003     *
004     * The contents of this file are subject to the terms of the Liferay Enterprise
005     * Subscription License ("License"). You may not use this file except in
006     * compliance with the License. You can obtain a copy of the License by
007     * contacting Liferay, Inc. See the License for the specific language governing
008     * permissions and limitations under the License, including but not limited to
009     * distribution rights of the Software.
010     *
011     *
012     *
013     */
014    
015    package com.liferay.portlet.documentlibrary.store;
016    
017    import com.liferay.portal.kernel.exception.PortalException;
018    import com.liferay.portal.kernel.exception.SystemException;
019    import com.liferay.portal.kernel.log.Log;
020    import com.liferay.portal.kernel.log.LogFactoryUtil;
021    import com.liferay.portal.kernel.util.CharPool;
022    import com.liferay.portal.kernel.util.DateUtil;
023    import com.liferay.portal.kernel.util.FileUtil;
024    import com.liferay.portal.kernel.util.GetterUtil;
025    import com.liferay.portal.kernel.util.LocaleUtil;
026    import com.liferay.portal.kernel.util.PropsKeys;
027    import com.liferay.portal.kernel.util.ReleaseInfo;
028    import com.liferay.portal.kernel.util.StreamUtil;
029    import com.liferay.portal.kernel.util.StringBundler;
030    import com.liferay.portal.kernel.util.StringPool;
031    import com.liferay.portal.kernel.util.SystemProperties;
032    import com.liferay.portal.kernel.util.Time;
033    import com.liferay.portal.kernel.util.Validator;
034    import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
035    import com.liferay.portal.util.PropsUtil;
036    import com.liferay.portal.util.PropsValues;
037    import com.liferay.portlet.documentlibrary.DuplicateFileException;
038    import com.liferay.portlet.documentlibrary.NoSuchFileException;
039    
040    import java.io.File;
041    import java.io.FileInputStream;
042    import java.io.FileOutputStream;
043    import java.io.IOException;
044    import java.io.InputStream;
045    import java.io.OutputStream;
046    
047    import java.util.ArrayList;
048    import java.util.Arrays;
049    import java.util.Date;
050    import java.util.List;
051    import java.util.Properties;
052    
053    import org.jets3t.service.Jets3tProperties;
054    import org.jets3t.service.S3Service;
055    import org.jets3t.service.S3ServiceException;
056    import org.jets3t.service.ServiceException;
057    import org.jets3t.service.impl.rest.httpclient.RestS3Service;
058    import org.jets3t.service.model.S3Bucket;
059    import org.jets3t.service.model.S3Object;
060    import org.jets3t.service.model.StorageObject;
061    import org.jets3t.service.security.AWSCredentials;
062    import org.jets3t.service.utils.MultipartUtils;
063    
064    /**
065     * @author Brian Wing Shun Chan
066     * @author Sten Martinez
067     * @author Edward Han
068     * @author Vilmos Papp
069     * @author Mate Thurzo
070     */
071    public class S3Store extends BaseStore {
072    
073            public S3Store() {
074                    try {
075                            _s3Service = getS3Service();
076                            _s3Bucket = getS3Bucket();
077                    }
078                    catch (S3ServiceException s3se) {
079                            _log.error(s3se.getMessage());
080                    }
081            }
082    
083            @Override
084            public void addDirectory(
085                    long companyId, long repositoryId, String dirName) {
086            }
087    
088            @Override
089            public void addFile(
090                            long companyId, long repositoryId, String fileName, InputStream is)
091                    throws SystemException {
092    
093                    try {
094                            File file = FileUtil.createTempFile(is);
095    
096                            addFile(companyId, repositoryId, fileName, file);
097                    }
098                    catch (IOException ioe) {
099                            throw new SystemException(ioe);
100                    }
101                    finally {
102                            StreamUtil.cleanUp(is);
103                    }
104            }
105    
106            @Override
107            public void addFile(
108                            long companyId, long repositoryId, String fileName, File file)
109                    throws SystemException {
110    
111                    putObject(
112                            getKey(companyId, repositoryId, fileName, VERSION_DEFAULT), file);
113            }
114    
115            @Override
116            public void checkRoot(long companyId) {
117            }
118    
119            @Override
120            public void deleteDirectory(
121                            long companyId, long repositoryId, String dirName)
122                    throws SystemException {
123    
124                    try {
125                            S3Object[] s3Objects = _s3Service.listObjects(
126                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
127                                    null);
128    
129                            for (S3Object s3Object : s3Objects) {
130                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
131                            }
132                    }
133                    catch (S3ServiceException s3se) {
134                            throw new SystemException(s3se);
135                    }
136            }
137    
138            @Override
139            public void deleteFile(long companyId, long repositoryId, String fileName)
140                    throws SystemException {
141    
142                    try {
143                            S3Object[] s3Objects = _s3Service.listObjects(
144                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
145                                    null);
146    
147                            for (S3Object s3Object : s3Objects) {
148                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
149                            }
150                    }
151                    catch (S3ServiceException s3se) {
152                            throw new SystemException(s3se);
153                    }
154            }
155    
156            @Override
157            public void deleteFile(
158                            long companyId, long repositoryId, String fileName,
159                            String versionLabel)
160                    throws SystemException {
161    
162                    try {
163                            _s3Service.deleteObject(
164                                    _s3Bucket,
165                                    getKey(companyId, repositoryId, fileName, versionLabel));
166                    }
167                    catch (S3ServiceException s3se) {
168                            throw new SystemException(s3se);
169                    }
170            }
171    
172            @Override
173            public File getFile(
174                            long companyId, long repositoryId, String fileName,
175                            String versionLabel)
176                    throws PortalException, SystemException {
177    
178                    try {
179                            if (Validator.isNull(versionLabel)) {
180                                    versionLabel = getHeadVersionLabel(
181                                            companyId, repositoryId, fileName);
182                            }
183    
184                            S3Object s3Object = _s3Service.getObject(
185                                    _s3Bucket.getName(),
186                                    getKey(companyId, repositoryId, fileName, versionLabel));
187    
188                            File tempFile = getTempFile(s3Object, fileName);
189    
190                            cleanUpTempFiles();
191    
192                            return tempFile;
193                    }
194                    catch (IOException ioe) {
195                            throw new SystemException(ioe);
196                    }
197                    catch (ServiceException se) {
198                            throw new SystemException(se);
199                    }
200            }
201    
202            @Override
203            public InputStream getFileAsStream(
204                            long companyId, long repositoryId, String fileName,
205                            String versionLabel)
206                    throws PortalException, SystemException {
207    
208                    try {
209                            if (Validator.isNull(versionLabel)) {
210                                    versionLabel = getHeadVersionLabel(
211                                            companyId, repositoryId, fileName);
212                            }
213    
214                            S3Object s3Object = _s3Service.getObject(
215                                    _s3Bucket.getName(),
216                                    getKey(companyId, repositoryId, fileName, versionLabel));
217    
218                            return s3Object.getDataInputStream();
219                    }
220                    catch (ServiceException se) {
221                            throw new SystemException(se);
222                    }
223            }
224    
225            @Override
226            public String[] getFileNames(long companyId, long repositoryId)
227                    throws SystemException {
228    
229                    try {
230                            S3Object[] s3Objects = _s3Service.listObjects(
231                                    _s3Bucket.getName(), getKey(companyId, repositoryId), null);
232    
233                            return getFileNames(s3Objects);
234                    }
235                    catch (S3ServiceException s3se) {
236                            throw new SystemException(s3se);
237                    }
238            }
239    
240            @Override
241            public String[] getFileNames(
242                            long companyId, long repositoryId, String dirName)
243                    throws SystemException {
244    
245                    try {
246                            S3Object[] s3Objects = _s3Service.listObjects(
247                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
248                                    null);
249    
250                            return getFileNames(s3Objects);
251                    }
252                    catch (S3ServiceException s3se) {
253                            throw new SystemException(s3se);
254                    }
255            }
256    
257            @Override
258            public long getFileSize(long companyId, long repositoryId, String fileName)
259                    throws PortalException, SystemException {
260    
261                    try {
262                            String versionLabel = getHeadVersionLabel(
263                                    companyId, repositoryId, fileName);
264    
265                            StorageObject storageObject = _s3Service.getObjectDetails(
266                                    _s3Bucket.getName(),
267                                    getKey(companyId, repositoryId, fileName, versionLabel));
268    
269                            return storageObject.getContentLength();
270                    }
271                    catch (ServiceException se) {
272                            throw new SystemException(se);
273                    }
274            }
275    
276            @Override
277            public boolean hasDirectory(
278                    long companyId, long repositoryId, String dirName) {
279    
280                    return true;
281            }
282    
283            @Override
284            public boolean hasFile(
285                            long companyId, long repositoryId, String fileName,
286                            String versionLabel)
287                    throws SystemException {
288    
289                    try {
290                            S3Object[] s3Objects = _s3Service.listObjects(
291                                    _s3Bucket.getName(),
292                                    getKey(companyId, repositoryId, fileName, versionLabel), null);
293    
294                            if (s3Objects.length == 0) {
295                                    return false;
296                            }
297                            else {
298                                    return true;
299                            }
300                    }
301                    catch (S3ServiceException s3se) {
302                            throw new SystemException(s3se);
303                    }
304            }
305    
306            @Override
307            public void move(String srcDir, String destDir) {
308            }
309    
310            @Override
311            public void updateFile(
312                            long companyId, long repositoryId, long newRepositoryId,
313                            String fileName)
314                    throws PortalException, SystemException {
315    
316                    File tempFile = null;
317                    InputStream is = null;
318                    S3Object newS3Object = null;
319    
320                    if (repositoryId == newRepositoryId) {
321                            throw new DuplicateFileException(
322                                    String.format(
323                                            "{companyId=%s, fileName=%s, repositoryId=%s}", companyId,
324                                            fileName, repositoryId));
325                    }
326    
327                    try {
328                            S3Object[] s3Objects = _s3Service.listObjects(
329                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
330                                    null);
331    
332                            for (S3Object oldS3Object : s3Objects) {
333                                    String oldKey = oldS3Object.getKey();
334    
335                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
336    
337                                    tempFile = new File(
338                                            SystemProperties.get(SystemProperties.TMP_DIR) +
339                                                    File.separator + PortalUUIDUtil.generate());
340    
341                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
342    
343                                    is = new FileInputStream(tempFile);
344    
345                                    String newPrefix = getKey(companyId, newRepositoryId);
346    
347                                    int x = oldKey.indexOf(CharPool.SLASH);
348    
349                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
350    
351                                    String newKey = newPrefix + oldKey.substring(x);
352    
353                                    putObject(newKey, tempFile);
354    
355                                    _s3Service.deleteObject(_s3Bucket, oldKey);
356                            }
357                    }
358                    catch (IOException ioe) {
359                            throw new SystemException(ioe);
360                    }
361                    catch (ServiceException se) {
362                            throw new SystemException(se);
363                    }
364                    finally {
365                            StreamUtil.cleanUp(is);
366    
367                            FileUtil.delete(tempFile);
368                    }
369            }
370    
371            @Override
372            public void updateFile(
373                            long companyId, long repositoryId, String fileName,
374                            String newFileName)
375                    throws PortalException, SystemException {
376    
377                    if (fileName.equals(newFileName)) {
378                            throw new DuplicateFileException(
379                                    String.format(
380                                            "{companyId=%s, fileName=%s, repositoryId=%s}", companyId,
381                                            fileName, repositoryId));
382                    }
383    
384                    File tempFile = null;
385                    InputStream is = null;
386                    S3Object newS3Object = null;
387    
388                    try {
389                            S3Object[] s3Objects = _s3Service.listObjects(
390                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
391                                    null);
392    
393                            for (S3Object oldS3Object : s3Objects) {
394                                    String oldKey = oldS3Object.getKey();
395    
396                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
397    
398                                    tempFile = new File(
399                                            SystemProperties.get(SystemProperties.TMP_DIR) +
400                                                    File.separator + PortalUUIDUtil.generate());
401    
402                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
403    
404                                    oldS3Object.closeDataInputStream();
405    
406                                    is = new FileInputStream(tempFile);
407    
408                                    String newPrefix = getKey(companyId, repositoryId, newFileName);
409    
410                                    int x = oldKey.indexOf(StringPool.SLASH);
411    
412                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
413                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
414    
415                                    String newKey = newPrefix + oldKey.substring(x);
416    
417                                    putObject(newKey, tempFile);
418    
419                                    _s3Service.deleteObject(_s3Bucket, oldKey);
420                            }
421                    }
422                    catch (IOException ioe) {
423                            throw new SystemException(ioe);
424                    }
425                    catch (ServiceException se) {
426                            throw new SystemException(se);
427                    }
428                    finally {
429                            StreamUtil.cleanUp(is);
430    
431                            FileUtil.delete(tempFile);
432                    }
433            }
434    
435            @Override
436            public void updateFile(
437                            long companyId, long repositoryId, String fileName,
438                            String versionLabel, InputStream is)
439                    throws SystemException {
440    
441                    File file = null;
442    
443                    try {
444                            file = FileUtil.createTempFile(is);
445    
446                            updateFile(companyId, repositoryId, fileName, versionLabel, file);
447                    }
448                    catch (Exception e) {
449                            throw new SystemException(e);
450                    }
451                    finally {
452                            StreamUtil.cleanUp(is);
453    
454                            FileUtil.delete(file);
455                    }
456            }
457    
458            @Override
459            public void updateFile(
460                            long companyId, long repositoryId, String fileName,
461                            String versionLabel, File file)
462                    throws PortalException, SystemException {
463    
464                    putObject(
465                            getKey(companyId, repositoryId, fileName, versionLabel), file);
466            }
467    
468            protected void cleanUpTempFiles() {
469                    _calledGetFileCount++;
470    
471                    if (_calledGetFileCount <
472                                    PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
473    
474                            return;
475                    }
476    
477                    synchronized (this) {
478                            if (_calledGetFileCount == 0) {
479                                    return;
480                            }
481    
482                            _calledGetFileCount = 0;
483    
484                            String tempDirName =
485                                    SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
486    
487                            File tempDir = new File(tempDirName);
488    
489                            long lastModified = System.currentTimeMillis();
490    
491                            lastModified -=
492                                    (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
493    
494                            cleanUpTempFiles(tempDir, lastModified);
495                    }
496            }
497    
498            protected void cleanUpTempFiles(File file, long lastModified) {
499                    if (!file.isDirectory()) {
500                            return;
501                    }
502    
503                    String[] fileNames = FileUtil.listDirs(file);
504    
505                    if (fileNames.length == 0) {
506                            if (file.lastModified() < lastModified) {
507                                    FileUtil.deltree(file);
508    
509                                    return;
510                            }
511                    }
512                    else {
513                            for (String fileName : fileNames) {
514                                    cleanUpTempFiles(new File(file, fileName), lastModified);
515                            }
516    
517                            String[] subfileNames = file.list();
518    
519                            if (subfileNames.length == 0) {
520                                    FileUtil.deltree(file);
521    
522                                    return;
523                            }
524                    }
525            }
526    
527            protected AWSCredentials getAWSCredentials() throws S3ServiceException {
528                    if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
529                            throw new S3ServiceException(
530                                    "S3 access and secret keys are not set");
531                    }
532                    else {
533                            return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
534                    }
535            }
536    
537            protected String getFileName(String key) {
538    
539                    // Convert /${companyId}/${repositoryId}/${dirName}/${fileName}
540                    // /${versionLabel} to /${dirName}/${fileName}
541    
542                    int x = key.indexOf(CharPool.SLASH);
543    
544                    x = key.indexOf(CharPool.SLASH, x + 1);
545    
546                    int y = key.lastIndexOf(CharPool.SLASH);
547    
548                    return key.substring(x, y);
549            }
550    
551            protected String[] getFileNames(S3Object[] s3Objects) {
552                    List<String> fileNames = new ArrayList<String>();
553    
554                    for (S3Object s3Object : s3Objects) {
555                            String fileName = getFileName(s3Object.getKey());
556    
557                            fileNames.add(fileName);
558                    }
559    
560                    return fileNames.toArray(new String[fileNames.size()]);
561            }
562    
563            protected String getHeadVersionLabel(
564                            long companyId, long repositoryId, String fileName)
565                    throws PortalException, S3ServiceException {
566    
567                    S3Object[] s3Objects = _s3Service.listObjects(
568                            _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
569                            null);
570    
571                    String[] keys = new String[s3Objects.length];
572    
573                    for (int i = 0; i < s3Objects.length; i++) {
574                            S3Object s3Object = s3Objects[i];
575    
576                            keys[i] = s3Object.getKey();
577                    }
578    
579                    if (keys.length > 0) {
580                            Arrays.sort(keys);
581    
582                            String headKey = keys[keys.length - 1];
583    
584                            int x = headKey.lastIndexOf(CharPool.SLASH);
585    
586                            return headKey.substring(x + 1);
587                    }
588                    else {
589                            throw new NoSuchFileException(fileName);
590                    }
591            }
592    
593            protected Jets3tProperties getJets3tProperties() {
594                    Jets3tProperties jets3tProperties = new Jets3tProperties();
595    
596                    jets3tProperties.loadAndReplaceProperties(_jets3tProperties, "liferay");
597    
598                    if (_log.isInfoEnabled()) {
599                            _log.info("Jets3t properties: " + jets3tProperties.getProperties());
600                    }
601    
602                    return jets3tProperties;
603            }
604    
605            protected String getKey(long companyId, long repositoryId) {
606                    StringBundler sb = new StringBundler(4);
607    
608                    sb.append(companyId);
609                    sb.append(StringPool.SLASH);
610                    sb.append(repositoryId);
611    
612                    return sb.toString();
613            }
614    
615            protected String getKey(
616                    long companyId, long repositoryId, String fileName) {
617    
618                    StringBundler sb = new StringBundler(4);
619    
620                    sb.append(companyId);
621                    sb.append(StringPool.SLASH);
622                    sb.append(repositoryId);
623                    sb.append(getNormalizedFileName(fileName));
624    
625                    return sb.toString();
626            }
627    
628            protected String getKey(
629                    long companyId, long repositoryId, String fileName,
630                    String versionLabel) {
631    
632                    StringBundler sb = new StringBundler(6);
633    
634                    sb.append(companyId);
635                    sb.append(StringPool.SLASH);
636                    sb.append(repositoryId);
637                    sb.append(getNormalizedFileName(fileName));
638                    sb.append(StringPool.SLASH);
639                    sb.append(versionLabel);
640    
641                    return sb.toString();
642            }
643    
644            protected String getNormalizedFileName(String fileName) {
645                    String normalizedFileName = fileName;
646    
647                    if (!fileName.startsWith(StringPool.SLASH)) {
648                            normalizedFileName = StringPool.SLASH + normalizedFileName;
649                    }
650    
651                    if (fileName.endsWith(StringPool.SLASH)) {
652                            normalizedFileName = normalizedFileName.substring(
653                                    0, normalizedFileName.length() - 1);
654                    }
655    
656                    return normalizedFileName;
657            }
658    
659            protected S3Bucket getS3Bucket() throws S3ServiceException {
660                    if (Validator.isNull(_BUCKET_NAME)) {
661                            throw new S3ServiceException("S3 bucket name is not set");
662                    }
663                    else {
664                            return getS3Service().getBucket(_BUCKET_NAME);
665                    }
666            }
667    
668            protected S3Service getS3Service() throws S3ServiceException {
669                    AWSCredentials credentials = getAWSCredentials();
670    
671                    Jets3tProperties jets3tProperties = getJets3tProperties();
672    
673                    return new RestS3Service(
674                            credentials, ReleaseInfo.getServerInfo(), null, jets3tProperties);
675            }
676    
677            protected File getTempFile(S3Object s3Object, String fileName)
678                    throws IOException, ServiceException {
679    
680                    StringBundler sb = new StringBundler(5);
681    
682                    sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
683                    sb.append(_TEMP_DIR_NAME);
684                    sb.append(
685                            DateUtil.getCurrentDate(
686                                    _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
687                    sb.append(getNormalizedFileName(fileName));
688    
689                    Date lastModifiedDate = s3Object.getLastModifiedDate();
690    
691                    sb.append(lastModifiedDate.getTime());
692    
693                    String tempFileName = sb.toString();
694    
695                    File tempFile = new File(tempFileName);
696    
697                    InputStream inputStream = s3Object.getDataInputStream();
698    
699                    if (tempFile.exists() &&
700                            (tempFile.lastModified() >= lastModifiedDate.getTime())) {
701    
702                            StreamUtil.cleanUp(inputStream);
703    
704                            return tempFile;
705                    }
706    
707                    if (inputStream == null) {
708                            throw new IOException("S3 object input stream is null");
709                    }
710    
711                    OutputStream outputStream = null;
712    
713                    try {
714                            File parentFile = tempFile.getParentFile();
715    
716                            FileUtil.mkdirs(parentFile);
717    
718                            outputStream = new FileOutputStream(tempFile);
719    
720                            StreamUtil.transfer(inputStream, outputStream);
721                    }
722                    finally {
723                            StreamUtil.cleanUp(inputStream, outputStream);
724                    }
725    
726                    return tempFile;
727            }
728    
729            protected void putObject(String key, File file) throws SystemException {
730                    try {
731                            MultipartUtils multipartUtils = new MultipartUtils(_PART_SIZE);
732    
733                            List<StorageObject> s3Objects = new ArrayList<StorageObject>();
734    
735                            S3Object s3Object = new S3Object(file);
736    
737                            s3Object.setBucketName(_BUCKET_NAME);
738                            s3Object.setKey(key);
739    
740                            s3Objects.add(s3Object);
741    
742                            multipartUtils.uploadObjects(
743                                    _BUCKET_NAME, _s3Service, s3Objects, null);
744                    }
745                    catch (Exception e) {
746                            throw new SystemException(e);
747                    }
748            }
749    
750            private static final String _ACCESS_KEY = PropsUtil.get(
751                    PropsKeys.DL_STORE_S3_ACCESS_KEY);
752    
753            private static final String _BUCKET_NAME = PropsUtil.get(
754                    PropsKeys.DL_STORE_S3_BUCKET_NAME);
755    
756            private static final long _PART_SIZE = GetterUtil.getLong(
757                    PropsUtil.get(PropsKeys.DL_STORE_S3_PART_SIZE));
758    
759            private static final String _SECRET_KEY = PropsUtil.get(
760                    PropsKeys.DL_STORE_S3_SECRET_KEY);
761    
762            private static final String _TEMP_DIR_NAME = "/liferay/s3";
763    
764            private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
765    
766            private static Log _log = LogFactoryUtil.getLog(S3Store.class);
767    
768            private static final Properties _jets3tProperties = PropsUtil.getProperties(
769                    PropsKeys.DL_STORE_S3_JETS3T, true);
770    
771            private int _calledGetFileCount;
772            private S3Bucket _s3Bucket;
773            private S3Service _s3Service;
774    
775    }