001    /**
002     * Copyright (c) 2000-present Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portlet.documentlibrary.store;
016    
017    import com.liferay.portal.kernel.exception.PortalException;
018    import com.liferay.portal.kernel.exception.SystemException;
019    import com.liferay.portal.kernel.log.Log;
020    import com.liferay.portal.kernel.log.LogFactoryUtil;
021    import com.liferay.portal.kernel.util.CharPool;
022    import com.liferay.portal.kernel.util.DateUtil;
023    import com.liferay.portal.kernel.util.FileUtil;
024    import com.liferay.portal.kernel.util.LocaleUtil;
025    import com.liferay.portal.kernel.util.PropsKeys;
026    import com.liferay.portal.kernel.util.StreamUtil;
027    import com.liferay.portal.kernel.util.StringBundler;
028    import com.liferay.portal.kernel.util.StringPool;
029    import com.liferay.portal.kernel.util.SystemProperties;
030    import com.liferay.portal.kernel.util.Time;
031    import com.liferay.portal.kernel.util.Validator;
032    import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033    import com.liferay.portal.util.PropsUtil;
034    import com.liferay.portal.util.PropsValues;
035    import com.liferay.portlet.documentlibrary.NoSuchFileException;
036    
037    import java.io.File;
038    import java.io.FileInputStream;
039    import java.io.FileOutputStream;
040    import java.io.IOException;
041    import java.io.InputStream;
042    import java.io.OutputStream;
043    
044    import java.util.ArrayList;
045    import java.util.Arrays;
046    import java.util.Date;
047    import java.util.List;
048    
049    import org.jets3t.service.S3Service;
050    import org.jets3t.service.S3ServiceException;
051    import org.jets3t.service.ServiceException;
052    import org.jets3t.service.impl.rest.httpclient.RestS3Service;
053    import org.jets3t.service.model.S3Bucket;
054    import org.jets3t.service.model.S3Object;
055    import org.jets3t.service.model.StorageObject;
056    import org.jets3t.service.security.AWSCredentials;
057    
058    /**
059     * @author Brian Wing Shun Chan
060     * @author Sten Martinez
061     * @author Edward Han
062     * @author Vilmos Papp
063     * @author Mate Thurzo
064     */
065    public class S3Store extends BaseStore {
066    
067            public S3Store() {
068                    try {
069                            _s3Service = getS3Service();
070                            _s3Bucket = getS3Bucket();
071                    }
072                    catch (S3ServiceException s3se) {
073                            _log.error(s3se.getMessage());
074                    }
075            }
076    
077            @Override
078            public void addDirectory(
079                    long companyId, long repositoryId, String dirName) {
080            }
081    
082            @Override
083            public void addFile(
084                    long companyId, long repositoryId, String fileName, InputStream is) {
085    
086                    try {
087                            S3Object s3Object = new S3Object(
088                                    _s3Bucket,
089                                    getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
090    
091                            s3Object.setDataInputStream(is);
092    
093                            _s3Service.putObject(_s3Bucket, s3Object);
094                    }
095                    catch (S3ServiceException s3se) {
096                            throw new SystemException(s3se);
097                    }
098                    finally {
099                            StreamUtil.cleanUp(is);
100                    }
101            }
102    
103            @Override
104            public void checkRoot(long companyId) {
105            }
106    
107            @Override
108            public void deleteDirectory(
109                    long companyId, long repositoryId, String dirName) {
110    
111                    try {
112                            S3Object[] s3Objects = _s3Service.listObjects(
113                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
114                                    null);
115    
116                            for (S3Object s3Object : s3Objects) {
117                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
118                            }
119                    }
120                    catch (S3ServiceException s3se) {
121                            throw new SystemException(s3se);
122                    }
123            }
124    
125            @Override
126            public void deleteFile(long companyId, long repositoryId, String fileName) {
127                    try {
128                            S3Object[] s3Objects = _s3Service.listObjects(
129                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
130                                    null);
131    
132                            for (S3Object s3Object : s3Objects) {
133                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
134                            }
135                    }
136                    catch (S3ServiceException s3se) {
137                            throw new SystemException(s3se);
138                    }
139            }
140    
141            @Override
142            public void deleteFile(
143                    long companyId, long repositoryId, String fileName,
144                    String versionLabel) {
145    
146                    try {
147                            _s3Service.deleteObject(
148                                    _s3Bucket,
149                                    getKey(companyId, repositoryId, fileName, versionLabel));
150                    }
151                    catch (S3ServiceException s3se) {
152                            throw new SystemException(s3se);
153                    }
154            }
155    
156            @Override
157            public File getFile(
158                            long companyId, long repositoryId, String fileName,
159                            String versionLabel)
160                    throws PortalException {
161    
162                    try {
163                            if (Validator.isNull(versionLabel)) {
164                                    versionLabel = getHeadVersionLabel(
165                                            companyId, repositoryId, fileName);
166                            }
167    
168                            S3Object s3Object = _s3Service.getObject(
169                                    _s3Bucket.getName(),
170                                    getKey(companyId, repositoryId, fileName, versionLabel));
171    
172                            File tempFile = getTempFile(s3Object, fileName);
173    
174                            cleanUpTempFiles();
175    
176                            return tempFile;
177                    }
178                    catch (IOException ioe) {
179                            throw new SystemException(ioe);
180                    }
181                    catch (ServiceException se) {
182                            throw new SystemException(se);
183                    }
184            }
185    
186            @Override
187            public InputStream getFileAsStream(
188                            long companyId, long repositoryId, String fileName,
189                            String versionLabel)
190                    throws PortalException {
191    
192                    try {
193                            if (Validator.isNull(versionLabel)) {
194                                    versionLabel = getHeadVersionLabel(
195                                            companyId, repositoryId, fileName);
196                            }
197    
198                            S3Object s3Object = _s3Service.getObject(
199                                    _s3Bucket.getName(),
200                                    getKey(companyId, repositoryId, fileName, versionLabel));
201    
202                            return s3Object.getDataInputStream();
203                    }
204                    catch (ServiceException se) {
205                            throw new SystemException(se);
206                    }
207            }
208    
209            @Override
210            public String[] getFileNames(long companyId, long repositoryId) {
211                    try {
212                            S3Object[] s3Objects = _s3Service.listObjects(
213                                    _s3Bucket.getName(), getKey(companyId, repositoryId), null);
214    
215                            return getFileNames(s3Objects);
216                    }
217                    catch (S3ServiceException s3se) {
218                            throw new SystemException(s3se);
219                    }
220            }
221    
222            @Override
223            public String[] getFileNames(
224                    long companyId, long repositoryId, String dirName) {
225    
226                    try {
227                            S3Object[] s3Objects = _s3Service.listObjects(
228                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
229                                    null);
230    
231                            return getFileNames(s3Objects);
232                    }
233                    catch (S3ServiceException s3se) {
234                            throw new SystemException(s3se);
235                    }
236            }
237    
238            @Override
239            public long getFileSize(long companyId, long repositoryId, String fileName)
240                    throws PortalException {
241    
242                    try {
243                            String versionLabel = getHeadVersionLabel(
244                                    companyId, repositoryId, fileName);
245    
246                            StorageObject storageObject = _s3Service.getObjectDetails(
247                                    _s3Bucket.getName(),
248                                    getKey(companyId, repositoryId, fileName, versionLabel));
249    
250                            return storageObject.getContentLength();
251                    }
252                    catch (ServiceException se) {
253                            throw new SystemException(se);
254                    }
255            }
256    
257            @Override
258            public boolean hasDirectory(
259                    long companyId, long repositoryId, String dirName) {
260    
261                    return true;
262            }
263    
264            @Override
265            public boolean hasFile(
266                    long companyId, long repositoryId, String fileName,
267                    String versionLabel) {
268    
269                    try {
270                            S3Object[] s3Objects = _s3Service.listObjects(
271                                    _s3Bucket.getName(),
272                                    getKey(companyId, repositoryId, fileName, versionLabel), null);
273    
274                            if (s3Objects.length == 0) {
275                                    return false;
276                            }
277                            else {
278                                    return true;
279                            }
280                    }
281                    catch (S3ServiceException s3se) {
282                            throw new SystemException(s3se);
283                    }
284            }
285    
286            @Override
287            public void move(String srcDir, String destDir) {
288            }
289    
290            @Override
291            public void updateFile(
292                    long companyId, long repositoryId, long newRepositoryId,
293                    String fileName) {
294    
295                    File tempFile = null;
296                    InputStream is = null;
297                    S3Object newS3Object = null;
298    
299                    try {
300                            S3Object[] s3Objects = _s3Service.listObjects(
301                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
302                                    null);
303    
304                            for (S3Object oldS3Object : s3Objects) {
305                                    String oldKey = oldS3Object.getKey();
306    
307                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
308    
309                                    tempFile = new File(
310                                            SystemProperties.get(SystemProperties.TMP_DIR) +
311                                                    File.separator + PortalUUIDUtil.generate());
312    
313                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
314    
315                                    is = new FileInputStream(tempFile);
316    
317                                    String newPrefix = getKey(companyId, newRepositoryId);
318    
319                                    int x = oldKey.indexOf(CharPool.SLASH);
320    
321                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
322    
323                                    String newKey = newPrefix + oldKey.substring(x);
324    
325                                    newS3Object = new S3Object(_s3Bucket, newKey);
326    
327                                    newS3Object.setDataInputStream(is);
328    
329                                    _s3Service.putObject(_s3Bucket, newS3Object);
330                                    _s3Service.deleteObject(_s3Bucket, oldKey);
331                            }
332                    }
333                    catch (IOException ioe) {
334                            throw new SystemException(ioe);
335                    }
336                    catch (ServiceException se) {
337                            throw new SystemException(se);
338                    }
339                    finally {
340                            StreamUtil.cleanUp(is);
341    
342                            FileUtil.delete(tempFile);
343                    }
344            }
345    
346            @Override
347            public void updateFile(
348                    long companyId, long repositoryId, String fileName,
349                    String newFileName) {
350    
351                    File tempFile = null;
352                    InputStream is = null;
353                    S3Object newS3Object = null;
354    
355                    try {
356                            S3Object[] s3Objects = _s3Service.listObjects(
357                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
358                                    null);
359    
360                            for (S3Object oldS3Object : s3Objects) {
361                                    String oldKey = oldS3Object.getKey();
362    
363                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
364    
365                                    tempFile = new File(
366                                            SystemProperties.get(SystemProperties.TMP_DIR) +
367                                                    File.separator + PortalUUIDUtil.generate());
368    
369                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
370    
371                                    oldS3Object.closeDataInputStream();
372    
373                                    is = new FileInputStream(tempFile);
374    
375                                    String newPrefix = getKey(companyId, repositoryId, newFileName);
376    
377                                    int x = oldKey.indexOf(StringPool.SLASH);
378    
379                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
380                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
381    
382                                    String newKey = newPrefix + oldKey.substring(x);
383    
384                                    newS3Object = new S3Object(_s3Bucket, newKey);
385    
386                                    newS3Object.setDataInputStream(is);
387    
388                                    _s3Service.putObject(_s3Bucket, newS3Object);
389                                    _s3Service.deleteObject(_s3Bucket, oldKey);
390                            }
391                    }
392                    catch (IOException ioe) {
393                            throw new SystemException(ioe);
394                    }
395                    catch (ServiceException se) {
396                            throw new SystemException(se);
397                    }
398                    finally {
399                            StreamUtil.cleanUp(is);
400    
401                            FileUtil.delete(tempFile);
402                    }
403            }
404    
405            @Override
406            public void updateFile(
407                    long companyId, long repositoryId, String fileName, String versionLabel,
408                    InputStream is) {
409    
410                    try {
411                            S3Object s3Object = new S3Object(
412                                    _s3Bucket,
413                                    getKey(companyId, repositoryId, fileName, versionLabel));
414    
415                            s3Object.setDataInputStream(is);
416    
417                            _s3Service.putObject(_s3Bucket, s3Object);
418                    }
419                    catch (S3ServiceException s3se) {
420                            throw new SystemException(s3se);
421                    }
422                    finally {
423                            StreamUtil.cleanUp(is);
424                    }
425            }
426    
427            protected void cleanUpTempFiles() {
428                    _calledGetFileCount++;
429    
430                    if (_calledGetFileCount <
431                                    PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
432    
433                            return;
434                    }
435    
436                    synchronized (this) {
437                            if (_calledGetFileCount == 0) {
438                                    return;
439                            }
440    
441                            _calledGetFileCount = 0;
442    
443                            String tempDirName =
444                                    SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
445    
446                            File tempDir = new File(tempDirName);
447    
448                            long lastModified = System.currentTimeMillis();
449    
450                            lastModified -=
451                                    (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
452    
453                            cleanUpTempFiles(tempDir, lastModified);
454                    }
455            }
456    
457            protected void cleanUpTempFiles(File file, long lastModified) {
458                    if (!file.isDirectory()) {
459                            return;
460                    }
461    
462                    String[] fileNames = FileUtil.listDirs(file);
463    
464                    if (fileNames.length == 0) {
465                            if (file.lastModified() < lastModified) {
466                                    FileUtil.deltree(file);
467    
468                                    return;
469                            }
470                    }
471                    else {
472                            for (String fileName : fileNames) {
473                                    cleanUpTempFiles(new File(file, fileName), lastModified);
474                            }
475    
476                            String[] subfileNames = file.list();
477    
478                            if (subfileNames.length == 0) {
479                                    FileUtil.deltree(file);
480    
481                                    return;
482                            }
483                    }
484            }
485    
486            protected AWSCredentials getAWSCredentials() throws S3ServiceException {
487                    if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
488                            throw new S3ServiceException(
489                                    "S3 access and secret keys are not set");
490                    }
491                    else {
492                            return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
493                    }
494            }
495    
496            protected String getFileName(String key) {
497    
498                    // Convert /${companyId}/${repositoryId}/${dirName}/${fileName}
499                    // /${versionLabel} to /${dirName}/${fileName}
500    
501                    int x = key.indexOf(CharPool.SLASH);
502    
503                    x = key.indexOf(CharPool.SLASH, x + 1);
504    
505                    int y = key.lastIndexOf(CharPool.SLASH);
506    
507                    return key.substring(x, y);
508            }
509    
510            protected String[] getFileNames(S3Object[] s3Objects) {
511                    List<String> fileNames = new ArrayList<String>();
512    
513                    for (S3Object s3Object : s3Objects) {
514                            String fileName = getFileName(s3Object.getKey());
515    
516                            fileNames.add(fileName);
517                    }
518    
519                    return fileNames.toArray(new String[fileNames.size()]);
520            }
521    
522            protected String getHeadVersionLabel(
523                            long companyId, long repositoryId, String fileName)
524                    throws PortalException, S3ServiceException {
525    
526                    S3Object[] s3Objects = _s3Service.listObjects(
527                            _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
528                            null);
529    
530                    String[] keys = new String[s3Objects.length];
531    
532                    for (int i = 0; i < s3Objects.length; i++) {
533                            S3Object s3Object = s3Objects[i];
534    
535                            keys[i] = s3Object.getKey();
536                    }
537    
538                    if (keys.length > 0) {
539                            Arrays.sort(keys);
540    
541                            String headKey = keys[keys.length - 1];
542    
543                            int x = headKey.lastIndexOf(CharPool.SLASH);
544    
545                            return headKey.substring(x + 1);
546                    }
547                    else {
548                            throw new NoSuchFileException(fileName);
549                    }
550            }
551    
552            protected String getKey(long companyId, long repositoryId) {
553                    return companyId + StringPool.SLASH + repositoryId;
554            }
555    
556            protected String getKey(
557                    long companyId, long repositoryId, String fileName) {
558    
559                    StringBundler sb = new StringBundler(4);
560    
561                    sb.append(companyId);
562                    sb.append(StringPool.SLASH);
563                    sb.append(repositoryId);
564                    sb.append(getNormalizedFileName(fileName));
565    
566                    return sb.toString();
567            }
568    
569            protected String getKey(
570                    long companyId, long repositoryId, String fileName,
571                    String versionLabel) {
572    
573                    StringBundler sb = new StringBundler(6);
574    
575                    sb.append(companyId);
576                    sb.append(StringPool.SLASH);
577                    sb.append(repositoryId);
578                    sb.append(getNormalizedFileName(fileName));
579                    sb.append(StringPool.SLASH);
580                    sb.append(versionLabel);
581    
582                    return sb.toString();
583            }
584    
585            protected String getNormalizedFileName(String fileName) {
586                    String normalizedFileName = fileName;
587    
588                    if (!fileName.startsWith(StringPool.SLASH)) {
589                            normalizedFileName = StringPool.SLASH + normalizedFileName;
590                    }
591    
592                    if (fileName.endsWith(StringPool.SLASH)) {
593                            normalizedFileName = normalizedFileName.substring(
594                                    0, normalizedFileName.length() - 1);
595                    }
596    
597                    return normalizedFileName;
598            }
599    
600            protected S3Bucket getS3Bucket() throws S3ServiceException {
601                    if (Validator.isNull(_BUCKET_NAME)) {
602                            throw new S3ServiceException("S3 bucket name is not set");
603                    }
604                    else {
605                            return getS3Service().getBucket(_BUCKET_NAME);
606                    }
607            }
608    
609            protected S3Service getS3Service() throws S3ServiceException {
610                    AWSCredentials credentials = getAWSCredentials();
611    
612                    return new RestS3Service(credentials);
613            }
614    
615            protected File getTempFile(S3Object s3Object, String fileName)
616                    throws IOException, ServiceException {
617    
618                    StringBundler sb = new StringBundler(5);
619    
620                    sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
621                    sb.append(_TEMP_DIR_NAME);
622                    sb.append(
623                            DateUtil.getCurrentDate(
624                                    _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
625                    sb.append(getNormalizedFileName(fileName));
626    
627                    Date lastModifiedDate = s3Object.getLastModifiedDate();
628    
629                    sb.append(lastModifiedDate.getTime());
630    
631                    String tempFileName = sb.toString();
632    
633                    File tempFile = new File(tempFileName);
634    
635                    if (tempFile.exists() &&
636                            (tempFile.lastModified() >= lastModifiedDate.getTime())) {
637    
638                            return tempFile;
639                    }
640    
641                    InputStream inputStream = s3Object.getDataInputStream();
642    
643                    if (inputStream == null) {
644                            throw new IOException("S3 object input stream is null");
645                    }
646    
647                    OutputStream outputStream = null;
648    
649                    try {
650                            File parentFile = tempFile.getParentFile();
651    
652                            parentFile.mkdirs();
653    
654                            outputStream = new FileOutputStream(tempFile);
655    
656                            StreamUtil.transfer(inputStream, outputStream);
657                    }
658                    finally {
659                            StreamUtil.cleanUp(inputStream, outputStream);
660                    }
661    
662                    return tempFile;
663            }
664    
665            private static final String _ACCESS_KEY = PropsUtil.get(
666                    PropsKeys.DL_STORE_S3_ACCESS_KEY);
667    
668            private static final String _BUCKET_NAME = PropsUtil.get(
669                    PropsKeys.DL_STORE_S3_BUCKET_NAME);
670    
671            private static final String _SECRET_KEY = PropsUtil.get(
672                    PropsKeys.DL_STORE_S3_SECRET_KEY);
673    
674            private static final String _TEMP_DIR_NAME = "/liferay/s3";
675    
676            private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
677    
678            private static Log _log = LogFactoryUtil.getLog(S3Store.class);
679    
680            private int _calledGetFileCount;
681            private S3Bucket _s3Bucket;
682            private S3Service _s3Service;
683    
684    }