001    /**
002     * Copyright (c) 2000-2013 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portlet.documentlibrary.store;
016    
017    import com.liferay.portal.kernel.exception.PortalException;
018    import com.liferay.portal.kernel.exception.SystemException;
019    import com.liferay.portal.kernel.log.Log;
020    import com.liferay.portal.kernel.log.LogFactoryUtil;
021    import com.liferay.portal.kernel.util.CharPool;
022    import com.liferay.portal.kernel.util.DateUtil;
023    import com.liferay.portal.kernel.util.FileUtil;
024    import com.liferay.portal.kernel.util.LocaleUtil;
025    import com.liferay.portal.kernel.util.PropsKeys;
026    import com.liferay.portal.kernel.util.StreamUtil;
027    import com.liferay.portal.kernel.util.StringBundler;
028    import com.liferay.portal.kernel.util.StringPool;
029    import com.liferay.portal.kernel.util.SystemProperties;
030    import com.liferay.portal.kernel.util.Time;
031    import com.liferay.portal.kernel.util.Validator;
032    import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033    import com.liferay.portal.util.PropsUtil;
034    import com.liferay.portal.util.PropsValues;
035    import com.liferay.portlet.documentlibrary.NoSuchFileException;
036    
037    import java.io.File;
038    import java.io.FileInputStream;
039    import java.io.FileOutputStream;
040    import java.io.IOException;
041    import java.io.InputStream;
042    import java.io.OutputStream;
043    
044    import java.util.ArrayList;
045    import java.util.Arrays;
046    import java.util.Date;
047    import java.util.List;
048    
049    import org.jets3t.service.S3Service;
050    import org.jets3t.service.S3ServiceException;
051    import org.jets3t.service.ServiceException;
052    import org.jets3t.service.impl.rest.httpclient.RestS3Service;
053    import org.jets3t.service.model.S3Bucket;
054    import org.jets3t.service.model.S3Object;
055    import org.jets3t.service.model.StorageObject;
056    import org.jets3t.service.security.AWSCredentials;
057    
058    /**
059     * @author Brian Wing Shun Chan
060     * @author Sten Martinez
061     * @author Edward Han
062     * @author Vilmos Papp
063     * @author Mate Thurzo
064     */
065    public class S3Store extends BaseStore {
066    
067            public S3Store() {
068                    try {
069                            _s3Service = getS3Service();
070                            _s3Bucket = getS3Bucket();
071                    }
072                    catch (S3ServiceException s3se) {
073                            _log.error(s3se.getMessage());
074                    }
075            }
076    
077            @Override
078            public void addDirectory(
079                    long companyId, long repositoryId, String dirName) {
080            }
081    
082            @Override
083            public void addFile(
084                            long companyId, long repositoryId, String fileName, InputStream is)
085                    throws SystemException {
086    
087                    try {
088                            S3Object s3Object = new S3Object(
089                                    _s3Bucket,
090                                    getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
091    
092                            s3Object.setDataInputStream(is);
093    
094                            _s3Service.putObject(_s3Bucket, s3Object);
095    
096                            is.close();
097                    }
098                    catch (IOException ioe) {
099                            throw new SystemException(ioe);
100                    }
101                    catch (S3ServiceException s3se) {
102                            throw new SystemException(s3se);
103                    }
104            }
105    
106            @Override
107            public void checkRoot(long companyId) {
108            }
109    
110            @Override
111            public void deleteDirectory(
112                            long companyId, long repositoryId, String dirName)
113                    throws SystemException {
114    
115                    try {
116                            S3Object[] s3Objects = _s3Service.listObjects(
117                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
118                                    null);
119    
120                            for (S3Object s3Object : s3Objects) {
121                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
122                            }
123                    }
124                    catch (S3ServiceException s3se) {
125                            throw new SystemException(s3se);
126                    }
127            }
128    
129            @Override
130            public void deleteFile(long companyId, long repositoryId, String fileName)
131                    throws SystemException {
132    
133                    try {
134                            S3Object[] s3Objects = _s3Service.listObjects(
135                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
136                                    null);
137    
138                            for (S3Object s3Object : s3Objects) {
139                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
140                            }
141                    }
142                    catch (S3ServiceException s3se) {
143                            throw new SystemException(s3se);
144                    }
145            }
146    
147            @Override
148            public void deleteFile(
149                            long companyId, long repositoryId, String fileName,
150                            String versionLabel)
151                    throws SystemException {
152    
153                    try {
154                            _s3Service.deleteObject(
155                                    _s3Bucket,
156                                    getKey(companyId, repositoryId, fileName, versionLabel));
157                    }
158                    catch (S3ServiceException s3se) {
159                            throw new SystemException(s3se);
160                    }
161            }
162    
163            @Override
164            public File getFile(
165                            long companyId, long repositoryId, String fileName,
166                            String versionLabel)
167                    throws PortalException, SystemException {
168    
169                    try {
170                            if (Validator.isNull(versionLabel)) {
171                                    versionLabel = getHeadVersionLabel(
172                                            companyId, repositoryId, fileName);
173                            }
174    
175                            S3Object s3Object = _s3Service.getObject(
176                                    _s3Bucket.getName(),
177                                    getKey(companyId, repositoryId, fileName, versionLabel));
178    
179                            File tempFile = getTempFile(s3Object, fileName);
180    
181                            cleanUpTempFiles();
182    
183                            return tempFile;
184                    }
185                    catch (IOException ioe) {
186                            throw new SystemException(ioe);
187                    }
188                    catch (ServiceException se) {
189                            throw new SystemException(se);
190                    }
191            }
192    
193            @Override
194            public InputStream getFileAsStream(
195                            long companyId, long repositoryId, String fileName,
196                            String versionLabel)
197                    throws PortalException, SystemException {
198    
199                    try {
200                            if (Validator.isNull(versionLabel)) {
201                                    versionLabel = getHeadVersionLabel(
202                                            companyId, repositoryId, fileName);
203                            }
204    
205                            S3Object s3Object = _s3Service.getObject(
206                                    _s3Bucket.getName(),
207                                    getKey(companyId, repositoryId, fileName, versionLabel));
208    
209                            return s3Object.getDataInputStream();
210                    }
211                    catch (ServiceException se) {
212                            throw new SystemException(se);
213                    }
214            }
215    
216            public String[] getFileNames(long companyId, long repositoryId)
217                    throws SystemException {
218    
219                    try {
220                            S3Object[] s3Objects = _s3Service.listObjects(
221                                    _s3Bucket.getName(), getKey(companyId, repositoryId), null);
222    
223                            return getFileNames(s3Objects);
224                    }
225                    catch (S3ServiceException s3se) {
226                            throw new SystemException(s3se);
227                    }
228            }
229    
230            @Override
231            public String[] getFileNames(
232                            long companyId, long repositoryId, String dirName)
233                    throws SystemException {
234    
235                    try {
236                            S3Object[] s3Objects = _s3Service.listObjects(
237                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
238                                    null);
239    
240                            return getFileNames(s3Objects);
241                    }
242                    catch (S3ServiceException s3se) {
243                            throw new SystemException(s3se);
244                    }
245            }
246    
247            @Override
248            public long getFileSize(long companyId, long repositoryId, String fileName)
249                    throws PortalException, SystemException {
250    
251                    try {
252                            String versionLabel = getHeadVersionLabel(
253                                    companyId, repositoryId, fileName);
254    
255                            StorageObject storageObject = _s3Service.getObjectDetails(
256                                    _s3Bucket.getName(),
257                                    getKey(companyId, repositoryId, fileName, versionLabel));
258    
259                            return storageObject.getContentLength();
260                    }
261                    catch (ServiceException se) {
262                            throw new SystemException(se);
263                    }
264            }
265    
266            @Override
267            public boolean hasDirectory(
268                    long companyId, long repositoryId, String dirName) {
269    
270                    return true;
271            }
272    
273            @Override
274            public boolean hasFile(
275                            long companyId, long repositoryId, String fileName,
276                            String versionLabel)
277                    throws SystemException {
278    
279                    try {
280                            S3Object[] s3Objects = _s3Service.listObjects(
281                                    _s3Bucket.getName(),
282                                    getKey(companyId, repositoryId, fileName, versionLabel), null);
283    
284                            if (s3Objects.length == 0) {
285                                    return false;
286                            }
287                            else {
288                                    return true;
289                            }
290                    }
291                    catch (S3ServiceException s3se) {
292                            throw new SystemException(s3se);
293                    }
294            }
295    
296            @Override
297            public void move(String srcDir, String destDir) {
298            }
299    
300            @Override
301            public void updateFile(
302                            long companyId, long repositoryId, long newRepositoryId,
303                            String fileName)
304                    throws SystemException {
305    
306                    try {
307                            S3Object[] s3Objects = _s3Service.listObjects(
308                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
309                                    null);
310    
311                            for (S3Object oldS3Object : s3Objects) {
312                                    String oldKey = oldS3Object.getKey();
313    
314                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
315    
316                                    File tempFile = new File(
317                                            SystemProperties.get(SystemProperties.TMP_DIR) +
318                                                    File.separator + PortalUUIDUtil.generate());
319    
320                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
321    
322                                    InputStream is = new FileInputStream(tempFile);
323    
324                                    String newPrefix = getKey(companyId, newRepositoryId);
325    
326                                    int x = oldKey.indexOf(CharPool.SLASH);
327    
328                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
329    
330                                    String newKey = newPrefix + oldKey.substring(x + 1);
331    
332                                    S3Object newS3Object = new S3Object(_s3Bucket, newKey);
333    
334                                    newS3Object.setDataInputStream(is);
335    
336                                    _s3Service.putObject(_s3Bucket, newS3Object);
337                                    _s3Service.deleteObject(_s3Bucket, oldKey);
338    
339                                    is.close();
340    
341                                    FileUtil.delete(tempFile);
342                            }
343                    }
344                    catch (IOException ioe) {
345                            throw new SystemException(ioe);
346                    }
347                    catch (ServiceException se) {
348                            throw new SystemException(se);
349                    }
350            }
351    
352            public void updateFile(
353                            long companyId, long repositoryId, String fileName,
354                            String newFileName)
355                    throws SystemException {
356    
357                    try {
358                            S3Object[] s3Objects = _s3Service.listObjects(
359                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
360                                    null);
361    
362                            for (S3Object oldS3Object : s3Objects) {
363                                    String oldKey = oldS3Object.getKey();
364    
365                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
366    
367                                    File tempFile = new File(
368                                            SystemProperties.get(SystemProperties.TMP_DIR) +
369                                                    File.separator + PortalUUIDUtil.generate());
370    
371                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
372    
373                                    oldS3Object.closeDataInputStream();
374    
375                                    InputStream is = new FileInputStream(tempFile);
376    
377                                    String newPrefix = getKey(companyId, repositoryId, newFileName);
378    
379                                    int x = oldKey.indexOf(StringPool.SLASH);
380    
381                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
382                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
383    
384                                    String newKey = newPrefix + oldKey.substring(x + 1);
385    
386                                    S3Object newS3Object = new S3Object(_s3Bucket, newKey);
387    
388                                    newS3Object.setDataInputStream(is);
389    
390                                    _s3Service.putObject(_s3Bucket, newS3Object);
391                                    _s3Service.deleteObject(_s3Bucket, oldKey);
392    
393                                    newS3Object.closeDataInputStream();
394    
395                                    is.close();
396    
397                                    FileUtil.delete(tempFile);
398                            }
399                    }
400                    catch (IOException ioe) {
401                            throw new SystemException(ioe);
402                    }
403                    catch (ServiceException se) {
404                            throw new SystemException(se);
405                    }
406            }
407    
408            @Override
409            public void updateFile(
410                            long companyId, long repositoryId, String fileName,
411                            String versionLabel, InputStream is)
412                    throws SystemException {
413    
414                    try {
415                            S3Object s3Object = new S3Object(
416                                    _s3Bucket,
417                                    getKey(companyId, repositoryId, fileName, versionLabel));
418    
419                            s3Object.setDataInputStream(is);
420    
421                            _s3Service.putObject(_s3Bucket, s3Object);
422    
423                            is.close();
424                    }
425                    catch (IOException ioe) {
426                            throw new SystemException(ioe);
427                    }
428                    catch (S3ServiceException s3se) {
429                            throw new SystemException(s3se);
430                    }
431            }
432    
433            protected void cleanUpTempFiles() {
434                    _calledGetFileCount++;
435    
436                    if (_calledGetFileCount <
437                                    PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
438    
439                            return;
440                    }
441    
442                    synchronized (this) {
443                            if (_calledGetFileCount == 0) {
444                                    return;
445                            }
446    
447                            _calledGetFileCount = 0;
448    
449                            String tempDirName =
450                                    SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
451    
452                            File tempDir = new File(tempDirName);
453    
454                            long lastModified = System.currentTimeMillis();
455    
456                            lastModified -=
457                                    (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
458    
459                            cleanUpTempFiles(tempDir, lastModified);
460                    }
461            }
462    
463            protected void cleanUpTempFiles(File file, long lastModified) {
464                    if (!file.isDirectory()) {
465                            return;
466                    }
467    
468                    String[] fileNames = FileUtil.listDirs(file);
469    
470                    if (fileNames.length == 0) {
471                            if (file.lastModified() < lastModified) {
472                                    FileUtil.deltree(file);
473    
474                                    return;
475                            }
476                    }
477                    else {
478                            for (String fileName : fileNames) {
479                                    cleanUpTempFiles(new File(file, fileName), lastModified);
480                            }
481    
482                            String[] subfileNames = file.list();
483    
484                            if (subfileNames.length == 0) {
485                                    FileUtil.deltree(file);
486    
487                                    return;
488                            }
489                    }
490            }
491    
492            protected AWSCredentials getAWSCredentials() throws S3ServiceException {
493                    if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
494                            throw new S3ServiceException(
495                                    "S3 access and secret keys are not set");
496                    }
497                    else {
498                            return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
499                    }
500            }
501    
502            protected String getFileName(String key) {
503    
504                    // Convert /${companyId}/${repositoryId}/${dirName}/${fileName}
505                    // /${versionLabel} to /${dirName}/${fileName}
506    
507                    int x = key.indexOf(CharPool.SLASH);
508    
509                    x = key.indexOf(CharPool.SLASH, x + 1);
510    
511                    int y = key.lastIndexOf(CharPool.SLASH);
512    
513                    return key.substring(x, y);
514            }
515    
516            protected String[] getFileNames(S3Object[] s3Objects) {
517                    List<String> fileNames = new ArrayList<String>();
518    
519                    for (S3Object s3Object : s3Objects) {
520                            String fileName = getFileName(s3Object.getKey());
521    
522                            fileNames.add(fileName);
523                    }
524    
525                    return fileNames.toArray(new String[fileNames.size()]);
526            }
527    
528            protected String getHeadVersionLabel(
529                            long companyId, long repositoryId, String fileName)
530                    throws PortalException, S3ServiceException {
531    
532                    S3Object[] s3Objects = _s3Service.listObjects(
533                            _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
534                            null);
535    
536                    String[] keys = new String[s3Objects.length];
537    
538                    for (int i = 0; i < s3Objects.length; i++) {
539                            S3Object s3Object = s3Objects[i];
540    
541                            keys[i] = s3Object.getKey();
542                    }
543    
544                    if (keys.length > 0) {
545                            Arrays.sort(keys);
546    
547                            String headKey = keys[keys.length - 1];
548    
549                            int x = headKey.lastIndexOf(CharPool.SLASH);
550    
551                            return headKey.substring(x + 1);
552                    }
553                    else {
554                            throw new NoSuchFileException(fileName);
555                    }
556            }
557    
558            protected String getKey(long companyId, long repositoryId) {
559                    StringBundler sb = new StringBundler(4);
560    
561                    sb.append(companyId);
562                    sb.append(StringPool.SLASH);
563                    sb.append(repositoryId);
564    
565                    return sb.toString();
566            }
567    
568            protected String getKey(
569                    long companyId, long repositoryId, String fileName) {
570    
571                    StringBundler sb = new StringBundler(4);
572    
573                    sb.append(companyId);
574                    sb.append(StringPool.SLASH);
575                    sb.append(repositoryId);
576                    sb.append(getNormalizedFileName(fileName));
577    
578                    return sb.toString();
579            }
580    
581            protected String getKey(
582                    long companyId, long repositoryId, String fileName,
583                    String versionLabel) {
584    
585                    StringBundler sb = new StringBundler(6);
586    
587                    sb.append(companyId);
588                    sb.append(StringPool.SLASH);
589                    sb.append(repositoryId);
590                    sb.append(getNormalizedFileName(fileName));
591                    sb.append(StringPool.SLASH);
592                    sb.append(versionLabel);
593    
594                    return sb.toString();
595            }
596    
597            protected String getNormalizedFileName(String fileName) {
598                    String normalizedFileName = fileName;
599    
600                    if (!fileName.startsWith(StringPool.SLASH)) {
601                            normalizedFileName = StringPool.SLASH + normalizedFileName;
602                    }
603    
604                    if (fileName.endsWith(StringPool.SLASH)) {
605                            normalizedFileName = normalizedFileName.substring(
606                                    0, normalizedFileName.length() - 1);
607                    }
608    
609                    return normalizedFileName;
610            }
611    
612            protected S3Bucket getS3Bucket() throws S3ServiceException {
613                    if (Validator.isNull(_BUCKET_NAME)) {
614                            throw new S3ServiceException("S3 bucket name is not set");
615                    }
616                    else {
617                            return getS3Service().getBucket(_BUCKET_NAME);
618                    }
619            }
620    
621            protected S3Service getS3Service() throws S3ServiceException {
622                    AWSCredentials credentials = getAWSCredentials();
623    
624                    return new RestS3Service(credentials);
625            }
626    
627            protected File getTempFile(S3Object s3Object, String fileName)
628                    throws IOException, ServiceException {
629    
630                    StringBundler sb = new StringBundler(5);
631    
632                    sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
633                    sb.append(_TEMP_DIR_NAME);
634                    sb.append(
635                            DateUtil.getCurrentDate(
636                                    _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
637                    sb.append(getNormalizedFileName(fileName));
638    
639                    Date lastModifiedDate = s3Object.getLastModifiedDate();
640    
641                    sb.append(lastModifiedDate.getTime());
642    
643                    String tempFileName = sb.toString();
644    
645                    File tempFile = new File(tempFileName);
646    
647                    if (tempFile.exists() &&
648                            (tempFile.lastModified() >= lastModifiedDate.getTime())) {
649    
650                            return tempFile;
651                    }
652    
653                    InputStream inputStream = s3Object.getDataInputStream();
654    
655                    if (inputStream == null) {
656                            throw new IOException("S3 object input stream is null");
657                    }
658    
659                    OutputStream outputStream = null;
660    
661                    try {
662                            File parentFile = tempFile.getParentFile();
663    
664                            parentFile.mkdirs();
665    
666                            outputStream = new FileOutputStream(tempFile);
667    
668                            StreamUtil.transfer(inputStream, outputStream);
669                    }
670                    finally {
671                            StreamUtil.cleanUp(inputStream, outputStream);
672                    }
673    
674                    return tempFile;
675            }
676    
677            private static final String _ACCESS_KEY = PropsUtil.get(
678                    PropsKeys.DL_STORE_S3_ACCESS_KEY);
679    
680            private static final String _BUCKET_NAME = PropsUtil.get(
681                    PropsKeys.DL_STORE_S3_BUCKET_NAME);
682    
683            private static final String _SECRET_KEY = PropsUtil.get(
684                    PropsKeys.DL_STORE_S3_SECRET_KEY);
685    
686            private static final String _TEMP_DIR_NAME = "/liferay/s3";
687    
688            private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
689    
690            private static Log _log = LogFactoryUtil.getLog(S3Store.class);
691    
692            private int _calledGetFileCount;
693            private S3Bucket _s3Bucket;
694            private S3Service _s3Service;
695    
696    }