001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.cache;
003
004import java.io.File;
005import java.io.IOException;
006import java.nio.channels.FileChannel;
007import java.nio.channels.FileLock;
008import java.nio.file.StandardOpenOption;
009import java.util.Arrays;
010import java.util.Properties;
011import java.util.logging.Handler;
012import java.util.logging.Level;
013import java.util.logging.LogRecord;
014import java.util.logging.Logger;
015import java.util.logging.SimpleFormatter;
016
017import org.apache.commons.jcs3.JCS;
018import org.apache.commons.jcs3.access.CacheAccess;
019import org.apache.commons.jcs3.auxiliary.AuxiliaryCache;
020import org.apache.commons.jcs3.auxiliary.AuxiliaryCacheFactory;
021import org.apache.commons.jcs3.auxiliary.disk.behavior.IDiskCacheAttributes;
022import org.apache.commons.jcs3.auxiliary.disk.block.BlockDiskCacheAttributes;
023import org.apache.commons.jcs3.auxiliary.disk.block.BlockDiskCacheFactory;
024import org.apache.commons.jcs3.auxiliary.disk.indexed.IndexedDiskCacheAttributes;
025import org.apache.commons.jcs3.auxiliary.disk.indexed.IndexedDiskCacheFactory;
026import org.apache.commons.jcs3.engine.CompositeCacheAttributes;
027import org.apache.commons.jcs3.engine.behavior.ICompositeCacheAttributes.DiskUsagePattern;
028import org.apache.commons.jcs3.engine.control.CompositeCache;
029import org.apache.commons.jcs3.utils.serialization.StandardSerializer;
030import org.openstreetmap.josm.data.preferences.BooleanProperty;
031import org.openstreetmap.josm.data.preferences.IntegerProperty;
032import org.openstreetmap.josm.spi.preferences.Config;
033import org.openstreetmap.josm.tools.Logging;
034import org.openstreetmap.josm.tools.Utils;
035
036/**
037 * Wrapper class for JCS Cache. Sets some sane environment and returns instances of cache objects.
038 * Static configuration for now assumes some small LRU cache in memory and larger LRU cache on disk
039 *
040 * @author Wiktor Niesiobędzki
041 * @since 8168
042 */
043public final class JCSCacheManager {
044    private static final long MAX_OBJECT_TTL = -1;
045    private static final String PREFERENCE_PREFIX = "jcs.cache";
046
047    /**
048     * Property that determines the disk cache implementation
049     */
050    public static final BooleanProperty USE_BLOCK_CACHE = new BooleanProperty(PREFERENCE_PREFIX + ".use_block_cache", true);
051
052    private static final AuxiliaryCacheFactory DISK_CACHE_FACTORY = getDiskCacheFactory();
053    private static FileLock cacheDirLock;
054
055    /**
056     * default objects to be held in memory by JCS caches (per region)
057     */
058    public static final IntegerProperty DEFAULT_MAX_OBJECTS_IN_MEMORY = new IntegerProperty(PREFERENCE_PREFIX + ".max_objects_in_memory", 1000);
059
060    private static final Logger jcsLog;
061
062    static {
063        // raising logging level gives ~500x performance gain
064        // http://westsworld.dk/blog/2008/01/jcs-and-performance/
065        jcsLog = Logger.getLogger("org.apache.commons.jcs3");
066        try {
067            jcsLog.setLevel(Level.INFO);
068            jcsLog.setUseParentHandlers(false);
069            // we need a separate handler from Main's, as we downgrade LEVEL.INFO to DEBUG level
070            Arrays.stream(jcsLog.getHandlers()).forEach(jcsLog::removeHandler);
071            jcsLog.addHandler(new Handler() {
072                final SimpleFormatter formatter = new SimpleFormatter();
073
074                @Override
075                public void publish(LogRecord record) {
076                    String msg = formatter.formatMessage(record);
077                    if (record.getLevel().intValue() >= Level.SEVERE.intValue()) {
078                        Logging.error(msg);
079                    } else if (record.getLevel().intValue() >= Level.WARNING.intValue()) {
080                        Logging.warn(msg);
081                        // downgrade INFO level to debug, as JCS is too verbose at INFO level
082                    } else if (record.getLevel().intValue() >= Level.INFO.intValue()) {
083                        Logging.debug(msg);
084                    } else {
085                        Logging.trace(msg);
086                    }
087                }
088
089                @Override
090                public void flush() {
091                    // nothing to be done on flush
092                }
093
094                @Override
095                public void close() {
096                    // nothing to be done on close
097                }
098            });
099        } catch (Exception e) {
100            Logging.log(Logging.LEVEL_ERROR, "Unable to configure JCS logs", e);
101        }
102    }
103
104    private JCSCacheManager() {
105        // Hide implicit public constructor for utility classes
106    }
107
108    static {
109        File cacheDir = new File(Config.getDirs().getCacheDirectory(true), "jcs");
110
111        try {
112            if (!cacheDir.exists() && !cacheDir.mkdirs()) {
113                Logging.warn("Cache directory " + cacheDir.toString() + " does not exists and could not create it");
114            } else {
115                File cacheDirLockPath = new File(cacheDir, ".lock");
116                try {
117                    if (!cacheDirLockPath.exists() && !cacheDirLockPath.createNewFile()) {
118                        Logging.warn("Cannot create cache dir lock file");
119                    }
120                    cacheDirLock = FileChannel.open(cacheDirLockPath.toPath(), StandardOpenOption.WRITE).tryLock();
121
122                    if (cacheDirLock == null)
123                        Logging.warn("Cannot lock cache directory. Will not use disk cache");
124                } catch (IOException e) {
125                    Logging.log(Logging.LEVEL_WARN, "Cannot create cache dir \"" + cacheDirLockPath + "\" lock file:", e);
126                    Logging.warn("Will not use disk cache");
127                }
128            }
129        } catch (Exception e) {
130            Logging.log(Logging.LEVEL_WARN, "Unable to configure disk cache. Will not use it", e);
131        }
132
133        // this could be moved to external file
134        Properties props = new Properties();
135        // these are default common to all cache regions
136        // use of auxiliary cache and sizing of the caches is done with giving proper getCache(...) params
137        // CHECKSTYLE.OFF: SingleSpaceSeparator
138        props.setProperty("jcs.default.cacheattributes",                      CompositeCacheAttributes.class.getCanonicalName());
139        props.setProperty("jcs.default.cacheattributes.MaxObjects",           DEFAULT_MAX_OBJECTS_IN_MEMORY.get().toString());
140        props.setProperty("jcs.default.cacheattributes.UseMemoryShrinker",    "true");
141        props.setProperty("jcs.default.cacheattributes.DiskUsagePatternName", "UPDATE"); // store elements on disk on put
142        props.setProperty("jcs.default.elementattributes",                    CacheEntryAttributes.class.getCanonicalName());
143        props.setProperty("jcs.default.elementattributes.IsEternal",          "false");
144        props.setProperty("jcs.default.elementattributes.MaxLife",            Long.toString(MAX_OBJECT_TTL));
145        props.setProperty("jcs.default.elementattributes.IdleTime",           Long.toString(MAX_OBJECT_TTL));
146        props.setProperty("jcs.default.elementattributes.IsSpool",            "true");
147        // CHECKSTYLE.ON: SingleSpaceSeparator
148        try {
149            JCS.setConfigProperties(props);
150        } catch (Exception e) {
151            Logging.log(Logging.LEVEL_WARN, "Unable to initialize JCS", e);
152        }
153    }
154
155    private static AuxiliaryCacheFactory getDiskCacheFactory() {
156        try {
157            return useBlockCache() ? new BlockDiskCacheFactory() : new IndexedDiskCacheFactory();
158        } catch (SecurityException | LinkageError e) {
159            Logging.error(e);
160            return null;
161        }
162    }
163
164    private static boolean useBlockCache() {
165        return Boolean.TRUE.equals(USE_BLOCK_CACHE.get());
166    }
167
168    /**
169     * Returns configured cache object for named cache region
170     * @param <K> key type
171     * @param <V> value type
172     * @param cacheName region name
173     * @return cache access object
174     */
175    public static <K, V> CacheAccess<K, V> getCache(String cacheName) {
176        return getCache(cacheName, DEFAULT_MAX_OBJECTS_IN_MEMORY.get().intValue(), 0, null);
177    }
178
179    /**
180     * Returns configured cache object with defined limits of memory cache and disk cache
181     * @param <K> key type
182     * @param <V> value type
183     * @param cacheName         region name
184     * @param maxMemoryObjects  number of objects to keep in memory
185     * @param maxDiskObjects    maximum size of the objects stored on disk in kB
186     * @param cachePath         path to disk cache. if null, no disk cache will be created
187     * @return cache access object
188     */
189    @SuppressWarnings("unchecked")
190    public static <K, V> CacheAccess<K, V> getCache(String cacheName, int maxMemoryObjects, int maxDiskObjects, String cachePath) {
191        CacheAccess<K, V> cacheAccess = getCacheAccess(cacheName, getCacheAttributes(maxMemoryObjects));
192
193        if (cachePath != null && cacheDirLock != null && cacheAccess != null && DISK_CACHE_FACTORY != null) {
194            CompositeCache<K, V> cc = cacheAccess.getCacheControl();
195            try {
196                IDiskCacheAttributes diskAttributes = getDiskCacheAttributes(maxDiskObjects, cachePath, cacheName);
197                if (cc.getAuxCaches().length == 0) {
198                    cc.setAuxCaches(new AuxiliaryCache[]{DISK_CACHE_FACTORY.createCache(
199                            diskAttributes, null, null, new StandardSerializer())});
200                }
201            } catch (Exception e) { // NOPMD
202                // in case any error in setting auxiliary cache, do not use disk cache at all - only memory
203                cc.setAuxCaches(new AuxiliaryCache[0]);
204                Logging.debug(e);
205            }
206        }
207        return cacheAccess;
208    }
209
210    private static <K, V> CacheAccess<K, V> getCacheAccess(String cacheName, CompositeCacheAttributes cacheAttributes) {
211        try {
212            return JCS.getInstance(cacheName, cacheAttributes);
213        } catch (SecurityException | LinkageError e) {
214            Logging.error(e);
215            return null;
216        }
217    }
218
219    /**
220     * Close all files to ensure, that all indexes and data are properly written
221     */
222    public static void shutdown() {
223        JCS.shutdown();
224    }
225
226    private static IDiskCacheAttributes getDiskCacheAttributes(int maxDiskObjects, String cachePath, String cacheName) {
227        IDiskCacheAttributes ret;
228        removeStaleFiles(cachePath + File.separator + cacheName, useBlockCache() ? "_INDEX_v2" : "_BLOCK_v2");
229        String newCacheName = cacheName + (useBlockCache() ? "_BLOCK_v2" : "_INDEX_v2");
230
231        if (useBlockCache()) {
232            BlockDiskCacheAttributes blockAttr = new BlockDiskCacheAttributes();
233            /*
234             * BlockDiskCache never optimizes the file, so when file size is reduced, it will never be truncated to desired size.
235             *
236             * If for some mysterious reason, file size is greater than the value set in preferences, just use the whole file. If the user
237             * wants to reduce the file size, (s)he may just go to preferences and there it should be handled (by removing old file)
238             */
239            File diskCacheFile = new File(cachePath + File.separator + newCacheName + ".data");
240            if (diskCacheFile.exists()) {
241                blockAttr.setMaxKeySize((int) Math.max(maxDiskObjects, diskCacheFile.length()/1024));
242            } else {
243                blockAttr.setMaxKeySize(maxDiskObjects);
244            }
245            blockAttr.setBlockSizeBytes(4096); // use 4k blocks
246            ret = blockAttr;
247        } else {
248            IndexedDiskCacheAttributes indexAttr = new IndexedDiskCacheAttributes();
249            indexAttr.setMaxKeySize(maxDiskObjects);
250            ret = indexAttr;
251        }
252        ret.setDiskLimitType(IDiskCacheAttributes.DiskLimitType.SIZE);
253        File path = new File(cachePath);
254        if (!path.exists() && !path.mkdirs()) {
255            Logging.warn("Failed to create cache path: {0}", cachePath);
256        } else {
257            ret.setDiskPath(cachePath);
258        }
259        ret.setCacheName(newCacheName);
260
261        return ret;
262    }
263
264    private static void removeStaleFiles(String basePathPart, String suffix) {
265        deleteCacheFiles(basePathPart + suffix);
266    }
267
268    private static void deleteCacheFiles(String basePathPart) {
269        Utils.deleteFileIfExists(new File(basePathPart + ".key"));
270        Utils.deleteFileIfExists(new File(basePathPart + ".data"));
271    }
272
273    private static CompositeCacheAttributes getCacheAttributes(int maxMemoryElements) {
274        CompositeCacheAttributes ret = new CompositeCacheAttributes();
275        ret.setMaxObjects(maxMemoryElements);
276        ret.setDiskUsagePattern(DiskUsagePattern.UPDATE);
277        return ret;
278    }
279}