001    /**
002     * Copyright (c) 2000-2012 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portlet.documentlibrary.store;
016    
017    import com.liferay.portal.kernel.exception.PortalException;
018    import com.liferay.portal.kernel.exception.SystemException;
019    import com.liferay.portal.kernel.log.Log;
020    import com.liferay.portal.kernel.log.LogFactoryUtil;
021    import com.liferay.portal.kernel.util.CharPool;
022    import com.liferay.portal.kernel.util.DateUtil;
023    import com.liferay.portal.kernel.util.FileUtil;
024    import com.liferay.portal.kernel.util.LocaleUtil;
025    import com.liferay.portal.kernel.util.PropsKeys;
026    import com.liferay.portal.kernel.util.StreamUtil;
027    import com.liferay.portal.kernel.util.StringBundler;
028    import com.liferay.portal.kernel.util.StringPool;
029    import com.liferay.portal.kernel.util.SystemProperties;
030    import com.liferay.portal.kernel.util.Time;
031    import com.liferay.portal.kernel.util.Validator;
032    import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033    import com.liferay.portal.util.PropsUtil;
034    import com.liferay.portal.util.PropsValues;
035    import com.liferay.portlet.documentlibrary.NoSuchFileException;
036    
037    import java.io.File;
038    import java.io.FileInputStream;
039    import java.io.FileOutputStream;
040    import java.io.IOException;
041    import java.io.InputStream;
042    import java.io.OutputStream;
043    
044    import java.util.ArrayList;
045    import java.util.Arrays;
046    import java.util.Date;
047    import java.util.List;
048    
049    import org.jets3t.service.S3Service;
050    import org.jets3t.service.S3ServiceException;
051    import org.jets3t.service.ServiceException;
052    import org.jets3t.service.impl.rest.httpclient.RestS3Service;
053    import org.jets3t.service.model.S3Bucket;
054    import org.jets3t.service.model.S3Object;
055    import org.jets3t.service.model.StorageObject;
056    import org.jets3t.service.security.AWSCredentials;
057    
058    /**
059     * @author Brian Wing Shun Chan
060     * @author Sten Martinez
061     * @author Edward Han
062     * @author Vilmos Papp
063     * @author Mate Thurzo
064     */
065    public class S3Store extends BaseStore {
066    
067            public S3Store() {
068                    try {
069                            _s3Service = getS3Service();
070                            _s3Bucket = getS3Bucket();
071                    }
072                    catch (S3ServiceException s3se) {
073                            _log.error(s3se.getMessage());
074                    }
075            }
076    
077            @Override
078            public void addDirectory(
079                    long companyId, long repositoryId, String dirName) {
080            }
081    
082            @Override
083            public void addFile(
084                            long companyId, long repositoryId, String fileName, InputStream is)
085                    throws SystemException {
086    
087                    try {
088                            S3Object s3Object = new S3Object(
089                                    _s3Bucket,
090                                    getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
091    
092                            s3Object.setDataInputStream(is);
093    
094                            _s3Service.putObject(_s3Bucket, s3Object);
095    
096                            is.close();
097                    }
098                    catch (IOException ioe) {
099                            throw new SystemException(ioe);
100                    }
101                    catch (S3ServiceException s3se) {
102                            throw new SystemException(s3se);
103                    }
104            }
105    
106            @Override
107            public void checkRoot(long companyId) {
108            }
109    
110            @Override
111            public void deleteDirectory(
112                            long companyId, long repositoryId, String dirName)
113                    throws SystemException {
114    
115                    try {
116                            S3Object[] s3Objects = _s3Service.listObjects(
117                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
118                                    null);
119    
120                            for (int i = 0; i < s3Objects.length; i++) {
121                                    S3Object s3Object = s3Objects[i];
122    
123                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
124                            }
125                    }
126                    catch (S3ServiceException s3se) {
127                            throw new SystemException(s3se);
128                    }
129            }
130    
131            @Override
132            public void deleteFile(long companyId, long repositoryId, String fileName)
133                    throws SystemException {
134    
135                    try {
136                            S3Object[] s3Objects = _s3Service.listObjects(
137                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
138                                    null);
139    
140                            for (int i = 0; i < s3Objects.length; i++) {
141                                    S3Object s3Object = s3Objects[i];
142    
143                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
144                            }
145                    }
146                    catch (S3ServiceException s3se) {
147                            throw new SystemException(s3se);
148                    }
149            }
150    
151            @Override
152            public void deleteFile(
153                            long companyId, long repositoryId, String fileName,
154                            String versionLabel)
155                    throws SystemException {
156    
157                    try {
158                            _s3Service.deleteObject(
159                                    _s3Bucket,
160                                    getKey(companyId, repositoryId, fileName, versionLabel));
161                    }
162                    catch (S3ServiceException s3se) {
163                            throw new SystemException(s3se);
164                    }
165            }
166    
167            @Override
168            public File getFile(
169                            long companyId, long repositoryId, String fileName,
170                            String versionLabel)
171                    throws PortalException, SystemException {
172    
173                    try {
174                            if (Validator.isNull(versionLabel)) {
175                                    versionLabel = getHeadVersionLabel(
176                                            companyId, repositoryId, fileName);
177                            }
178    
179                            S3Object s3Object = _s3Service.getObject(
180                                    _s3Bucket.getName(),
181                                    getKey(companyId, repositoryId, fileName, versionLabel));
182    
183                            File tempFile = getTempFile(s3Object, fileName);
184    
185                            cleanUpTempFiles();
186    
187                            return tempFile;
188                    }
189                    catch (IOException ioe) {
190                            throw new SystemException(ioe);
191                    }
192                    catch (ServiceException se) {
193                            throw new SystemException(se);
194                    }
195            }
196    
197            @Override
198            public InputStream getFileAsStream(
199                            long companyId, long repositoryId, String fileName,
200                            String versionLabel)
201                    throws PortalException, SystemException {
202    
203                    try {
204                            if (Validator.isNull(versionLabel)) {
205                                    versionLabel = getHeadVersionLabel(
206                                            companyId, repositoryId, fileName);
207                            }
208    
209                            S3Object s3Object = _s3Service.getObject(
210                                    _s3Bucket.getName(),
211                                    getKey(companyId, repositoryId, fileName, versionLabel));
212    
213                            return s3Object.getDataInputStream();
214                    }
215                    catch (ServiceException se) {
216                            throw new SystemException(se);
217                    }
218            }
219    
220            public String[] getFileNames(long companyId, long repositoryId)
221                    throws SystemException {
222    
223                    List<String> fileNames = new ArrayList<String>();
224    
225                    try {
226                            S3Object[] searchObjects = _s3Service.listObjects(
227                                    _s3Bucket.getName(), getKey(companyId, repositoryId), null);
228    
229                            for (int i = 0; i < searchObjects.length; i++) {
230                                    S3Object currentObject = searchObjects[i];
231    
232                                    String fileName = getFileName(currentObject.getKey());
233    
234                                    fileNames.add(fileName);
235                            }
236                    }
237                    catch (S3ServiceException s3se) {
238                            throw new SystemException(s3se);
239                    }
240    
241                    return fileNames.toArray(new String[fileNames.size()]);
242            }
243    
244            @Override
245            public String[] getFileNames(
246                            long companyId, long repositoryId, String dirName)
247                    throws SystemException {
248    
249                    try {
250                            List<String> list = new ArrayList<String>();
251    
252                            S3Object[] s3Objects = _s3Service.listObjects(
253                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
254                                    null);
255    
256                            for (int i = 0; i < s3Objects.length; i++) {
257                                    S3Object s3Object = s3Objects[i];
258    
259                                    // Convert /${companyId}/${repositoryId}/${dirName}/${fileName}
260                                    // /${versionLabel} to /${dirName}/${fileName}
261    
262                                    String key = s3Object.getKey();
263    
264                                    int x = key.indexOf(CharPool.SLASH);
265    
266                                    x = key.indexOf(CharPool.SLASH, x + 1);
267    
268                                    int y = key.lastIndexOf(CharPool.SLASH);
269    
270                                    list.add(key.substring(x, y));
271                            }
272    
273                            return list.toArray(new String[list.size()]);
274                    }
275                    catch (S3ServiceException s3se) {
276                            throw new SystemException(s3se);
277                    }
278            }
279    
280            @Override
281            public long getFileSize(long companyId, long repositoryId, String fileName)
282                    throws PortalException, SystemException {
283    
284                    try {
285                            String versionLabel = getHeadVersionLabel(
286                                    companyId, repositoryId, fileName);
287    
288                            StorageObject storageObject = _s3Service.getObjectDetails(
289                                    _s3Bucket.getName(),
290                                    getKey(companyId, repositoryId, fileName, versionLabel));
291    
292                            return storageObject.getContentLength();
293                    }
294                    catch (ServiceException se) {
295                            throw new SystemException(se);
296                    }
297            }
298    
299            @Override
300            public boolean hasDirectory(
301                    long companyId, long repositoryId, String dirName) {
302    
303                    return true;
304            }
305    
306            @Override
307            public boolean hasFile(
308                            long companyId, long repositoryId, String fileName,
309                            String versionLabel)
310                    throws SystemException {
311    
312                    try {
313                            S3Object[] s3Objects = _s3Service.listObjects(
314                                    _s3Bucket.getName(),
315                                    getKey(companyId, repositoryId, fileName, versionLabel), null);
316    
317                            if (s3Objects.length == 0) {
318                                    return false;
319                            }
320                            else {
321                                    return true;
322                            }
323                    }
324                    catch (S3ServiceException s3se) {
325                            throw new SystemException(s3se);
326                    }
327            }
328    
329            @Override
330            public void move(String srcDir, String destDir) {
331            }
332    
333            @Override
334            public void updateFile(
335                            long companyId, long repositoryId, long newRepositoryId,
336                            String fileName)
337                    throws SystemException {
338    
339                    try {
340                            S3Object[] s3Objects = _s3Service.listObjects(
341                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
342                                    null);
343    
344                            for (int i = 0; i < s3Objects.length; i++) {
345                                    S3Object oldS3Object = s3Objects[i];
346    
347                                    String oldKey = oldS3Object.getKey();
348    
349                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
350    
351                                    File tempFile = new File(
352                                            SystemProperties.get(SystemProperties.TMP_DIR) +
353                                                    File.separator + PortalUUIDUtil.generate());
354    
355                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
356    
357                                    InputStream is = new FileInputStream(tempFile);
358    
359                                    String newPrefix = getKey(companyId, newRepositoryId);
360    
361                                    int x = oldKey.indexOf(CharPool.SLASH);
362    
363                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
364    
365                                    String newKey = newPrefix + oldKey.substring(x + 1);
366    
367                                    S3Object newS3Object = new S3Object(_s3Bucket, newKey);
368    
369                                    newS3Object.setDataInputStream(is);
370    
371                                    _s3Service.putObject(_s3Bucket, newS3Object);
372                                    _s3Service.deleteObject(_s3Bucket, oldKey);
373    
374                                    is.close();
375    
376                                    FileUtil.delete(tempFile);
377                            }
378                    }
379                    catch (IOException ioe) {
380                            throw new SystemException(ioe);
381                    }
382                    catch (ServiceException se) {
383                            throw new SystemException(se);
384                    }
385            }
386    
387            public void updateFile(
388                            long companyId, long repositoryId, String fileName,
389                            String newFileName)
390                    throws SystemException {
391    
392                    try {
393                            S3Object[] s3Objects = _s3Service.listObjects(
394                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
395                                    null);
396    
397                            for (int i = 0; i < s3Objects.length; i++) {
398                                    S3Object oldS3Object = s3Objects[i];
399    
400                                    String oldKey = oldS3Object.getKey();
401    
402                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
403    
404                                    File tempFile = new File(
405                                            SystemProperties.get(SystemProperties.TMP_DIR) +
406                                                    File.separator + PortalUUIDUtil.generate());
407    
408                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
409    
410                                    oldS3Object.closeDataInputStream();
411    
412                                    InputStream is = new FileInputStream(tempFile);
413    
414                                    String newPrefix = getKey(companyId, repositoryId, newFileName);
415    
416                                    int x = oldKey.indexOf(StringPool.SLASH);
417    
418                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
419                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
420    
421                                    String newKey = newPrefix + oldKey.substring(x + 1);
422    
423                                    S3Object newS3Object = new S3Object(_s3Bucket, newKey);
424    
425                                    newS3Object.setDataInputStream(is);
426    
427                                    _s3Service.putObject(_s3Bucket, newS3Object);
428                                    _s3Service.deleteObject(_s3Bucket, oldKey);
429    
430                                    newS3Object.closeDataInputStream();
431    
432                                    is.close();
433    
434                                    FileUtil.delete(tempFile);
435                            }
436                    }
437                    catch (IOException ioe) {
438                            throw new SystemException(ioe);
439                    }
440                    catch (ServiceException se) {
441                            throw new SystemException(se);
442                    }
443            }
444    
445            @Override
446            public void updateFile(
447                            long companyId, long repositoryId, String fileName,
448                            String versionLabel, InputStream is)
449                    throws SystemException {
450    
451                    try {
452                            S3Object s3Object = new S3Object(
453                                    _s3Bucket,
454                                    getKey(companyId, repositoryId, fileName, versionLabel));
455    
456                            s3Object.setDataInputStream(is);
457    
458                            _s3Service.putObject(_s3Bucket, s3Object);
459    
460                            is.close();
461                    }
462                    catch (IOException ioe) {
463                            throw new SystemException(ioe);
464                    }
465                    catch (S3ServiceException s3se) {
466                            throw new SystemException(s3se);
467                    }
468            }
469    
470            protected void cleanUpTempFiles() {
471                    _calledGetFileCount++;
472    
473                    if (_calledGetFileCount <
474                                    PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
475    
476                            return;
477                    }
478    
479                    synchronized (this) {
480                            if (_calledGetFileCount == 0) {
481                                    return;
482                            }
483    
484                            _calledGetFileCount = 0;
485    
486                            String tempDirName =
487                                    SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
488    
489                            File tempDir = new File(tempDirName);
490    
491                            long lastModified = System.currentTimeMillis();
492    
493                            lastModified -=
494                                    (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
495    
496                            cleanUpTempFiles(tempDir, lastModified);
497                    }
498            }
499    
500            protected void cleanUpTempFiles(File file, long lastModified) {
501                    if (!file.isDirectory()) {
502                            return;
503                    }
504    
505                    String[] fileNames = FileUtil.listDirs(file);
506    
507                    if (fileNames.length == 0) {
508                            if (file.lastModified() < lastModified) {
509                                    FileUtil.deltree(file);
510    
511                                    return;
512                            }
513                    }
514                    else {
515                            for (String fileName : fileNames) {
516                                    cleanUpTempFiles(new File(file, fileName), lastModified);
517                            }
518    
519                            String[] subfileNames = file.list();
520    
521                            if (subfileNames.length == 0) {
522                                    FileUtil.deltree(file);
523    
524                                    return;
525                            }
526                    }
527            }
528    
529            protected AWSCredentials getAWSCredentials() throws S3ServiceException {
530                    if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
531                            throw new S3ServiceException(
532                                    "S3 access and secret keys are not set");
533                    }
534                    else {
535                            return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
536                    }
537            }
538    
539            protected String getFileName(String key) {
540                    int x = key.indexOf(CharPool.SLASH);
541    
542                    x = key.indexOf(CharPool.SLASH, x + 1);
543    
544                    int y = key.lastIndexOf(CharPool.SLASH);
545    
546                    return key.substring(x + 1, y);
547            }
548    
549            protected String getHeadVersionLabel(
550                            long companyId, long repositoryId, String fileName)
551                    throws PortalException, S3ServiceException {
552    
553                    S3Object[] s3Objects = _s3Service.listObjects(
554                            _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
555                            null);
556    
557                    String[] keys = new String[s3Objects.length];
558    
559                    for (int i = 0; i < s3Objects.length; i++) {
560                            S3Object s3Object = s3Objects[i];
561    
562                            keys[i] = s3Object.getKey();
563                    }
564    
565                    if (keys.length > 0) {
566                            Arrays.sort(keys);
567    
568                            String headKey = keys[keys.length - 1];
569    
570                            int x = headKey.lastIndexOf(CharPool.SLASH);
571    
572                            return headKey.substring(x + 1);
573                    }
574                    else {
575                            throw new NoSuchFileException(fileName);
576                    }
577            }
578    
579            protected String getKey(long companyId, long repositoryId) {
580                    StringBundler sb = new StringBundler(4);
581    
582                    sb.append(companyId);
583                    sb.append(StringPool.SLASH);
584                    sb.append(repositoryId);
585                    sb.append(StringPool.SLASH);
586    
587                    return sb.toString();
588            }
589    
590            protected String getKey(
591                    long companyId, long repositoryId, String fileName) {
592    
593                    StringBundler sb = new StringBundler(6);
594    
595                    sb.append(companyId);
596                    sb.append(StringPool.SLASH);
597                    sb.append(repositoryId);
598                    sb.append(StringPool.SLASH);
599                    sb.append(fileName);
600                    sb.append(StringPool.SLASH);
601    
602                    return sb.toString();
603            }
604    
605            protected String getKey(
606                    long companyId, long repositoryId, String fileName,
607                    String versionLabel) {
608    
609                    StringBundler sb = new StringBundler(7);
610    
611                    sb.append(companyId);
612                    sb.append(StringPool.SLASH);
613                    sb.append(repositoryId);
614                    sb.append(StringPool.SLASH);
615                    sb.append(fileName);
616                    sb.append(StringPool.SLASH);
617                    sb.append(versionLabel);
618    
619                    return sb.toString();
620            }
621    
622            protected S3Bucket getS3Bucket() throws S3ServiceException {
623                    if (Validator.isNull(_BUCKET_NAME)) {
624                            throw new S3ServiceException("S3 bucket name is not set");
625                    }
626                    else {
627                            return getS3Service().getBucket(_BUCKET_NAME);
628                    }
629            }
630    
631            protected S3Service getS3Service() throws S3ServiceException {
632                    AWSCredentials credentials = getAWSCredentials();
633    
634                    return new RestS3Service(credentials);
635            }
636    
637            protected File getTempFile(S3Object s3Object, String fileName)
638                    throws IOException, ServiceException {
639    
640                    StringBundler sb = new StringBundler(5);
641    
642                    sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
643                    sb.append(_TEMP_DIR_NAME);
644                    sb.append(
645                            DateUtil.getCurrentDate(
646                                    _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
647                    sb.append(fileName);
648    
649                    Date lastModifiedDate = s3Object.getLastModifiedDate();
650    
651                    sb.append(lastModifiedDate.getTime());
652    
653                    String tempFileName = sb.toString();
654    
655                    File tempFile = new File(tempFileName);
656    
657                    if (tempFile.exists() &&
658                            (tempFile.lastModified() >= lastModifiedDate.getTime())) {
659    
660                            return tempFile;
661                    }
662    
663                    InputStream inputStream = s3Object.getDataInputStream();
664    
665                    if (inputStream == null) {
666                            throw new IOException("S3 object input stream is null");
667                    }
668    
669                    OutputStream outputStream = null;
670    
671                    try {
672                            File parentFile = tempFile.getParentFile();
673    
674                            parentFile.mkdirs();
675    
676                            outputStream = new FileOutputStream(tempFile);
677    
678                            StreamUtil.transfer(inputStream, outputStream);
679                    }
680                    finally {
681                            StreamUtil.cleanUp(inputStream, outputStream);
682                    }
683    
684                    return tempFile;
685            }
686    
687            private static final String _ACCESS_KEY = PropsUtil.get(
688                    PropsKeys.DL_STORE_S3_ACCESS_KEY);
689    
690            private static final String _BUCKET_NAME = PropsUtil.get(
691                    PropsKeys.DL_STORE_S3_BUCKET_NAME);
692    
693            private static final String _SECRET_KEY = PropsUtil.get(
694                    PropsKeys.DL_STORE_S3_SECRET_KEY);
695    
696            private static final String _TEMP_DIR_NAME = "/liferay/s3";
697    
698            private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
699    
700            private static Log _log = LogFactoryUtil.getLog(S3Store.class);
701    
702            private int _calledGetFileCount;
703            private S3Bucket _s3Bucket;
704            private S3Service _s3Service;
705    
706    }