001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.gui.layer;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006import static org.openstreetmap.josm.tools.Utils.getSystemProperty;
007
008import java.io.BufferedReader;
009import java.io.File;
010import java.io.FileFilter;
011import java.io.IOException;
012import java.lang.management.ManagementFactory;
013import java.nio.charset.StandardCharsets;
014import java.nio.file.Files;
015import java.nio.file.Path;
016import java.time.Instant;
017import java.util.ArrayList;
018import java.util.Arrays;
019import java.util.Collections;
020import java.util.Comparator;
021import java.util.Date;
022import java.util.Deque;
023import java.util.HashSet;
024import java.util.Iterator;
025import java.util.LinkedList;
026import java.util.List;
027import java.util.Locale;
028import java.util.Set;
029import java.util.Timer;
030import java.util.TimerTask;
031import java.util.concurrent.ExecutionException;
032import java.util.concurrent.Future;
033import java.util.concurrent.TimeUnit;
034import java.util.regex.Pattern;
035
036import org.openstreetmap.josm.actions.OpenFileAction.OpenFileTask;
037import org.openstreetmap.josm.data.Data;
038import org.openstreetmap.josm.data.osm.NoteData;
039import org.openstreetmap.josm.data.osm.NoteData.NoteDataUpdateListener;
040import org.openstreetmap.josm.data.osm.event.AbstractDatasetChangedEvent;
041import org.openstreetmap.josm.data.osm.event.DataSetListenerAdapter;
042import org.openstreetmap.josm.data.osm.event.DataSetListenerAdapter.Listener;
043import org.openstreetmap.josm.data.preferences.BooleanProperty;
044import org.openstreetmap.josm.data.preferences.IntegerProperty;
045import org.openstreetmap.josm.gui.MainApplication;
046import org.openstreetmap.josm.gui.Notification;
047import org.openstreetmap.josm.gui.io.importexport.NoteImporter;
048import org.openstreetmap.josm.gui.io.importexport.OsmImporter;
049import org.openstreetmap.josm.gui.layer.LayerManager.LayerAddEvent;
050import org.openstreetmap.josm.gui.layer.LayerManager.LayerChangeListener;
051import org.openstreetmap.josm.gui.layer.LayerManager.LayerOrderChangeEvent;
052import org.openstreetmap.josm.gui.layer.LayerManager.LayerRemoveEvent;
053import org.openstreetmap.josm.gui.util.GuiHelper;
054import org.openstreetmap.josm.spi.preferences.Config;
055import org.openstreetmap.josm.tools.Logging;
056import org.openstreetmap.josm.tools.Utils;
057
058/**
059 * Saves data and note layers periodically so they can be recovered in case of a crash.
060 *
061 * There are 2 directories
062 *  - autosave dir: copies of the currently open data layers are saved here every
063 *      PROP_INTERVAL seconds. When a data layer is closed normally, the corresponding
064 *      files are removed. If this dir is non-empty on start, JOSM assumes
065 *      that it crashed last time.
066 *  - deleted layers dir: "secondary archive" - when autosaved layers are restored
067 *      they are copied to this directory. We cannot keep them in the autosave folder,
068 *      but just deleting it would be dangerous: Maybe a feature inside the file
069 *      caused JOSM to crash. If the data is valuable, the user can still try to
070 *      open with another versions of JOSM or fix the problem manually.
071 *
072 *      The deleted layers dir keeps at most PROP_DELETED_LAYERS files.
073 *
074 * @since  3378 (creation)
075 * @since 10386 (new LayerChangeListener interface)
076 */
077public class AutosaveTask extends TimerTask implements LayerChangeListener, Listener, NoteDataUpdateListener {
078
079    private static final char[] ILLEGAL_CHARACTERS = {'/', '\n', '\r', '\t', '\0', '\f', '`', '?', '*', '\\', '<', '>', '|', '\"', ':'};
080    private static final String AUTOSAVE_DIR = "autosave";
081    private static final String DELETED_LAYERS_DIR = "autosave/deleted_layers";
082
083    /**
084     * If autosave is enabled
085     */
086    public static final BooleanProperty PROP_AUTOSAVE_ENABLED = new BooleanProperty("autosave.enabled", true);
087    /**
088     * The number of files to store per layer
089     */
090    public static final IntegerProperty PROP_FILES_PER_LAYER = new IntegerProperty("autosave.filesPerLayer", 1);
091    /**
092     * How many deleted layers should be stored
093     */
094    public static final IntegerProperty PROP_DELETED_LAYERS = new IntegerProperty("autosave.deletedLayersBackupCount", 5);
095    /**
096     * The autosave interval, in seconds
097     */
098    public static final IntegerProperty PROP_INTERVAL = new IntegerProperty("autosave.interval", (int) TimeUnit.MINUTES.toSeconds(5));
099    /**
100     * The maximum number of autosave files to store
101     */
102    public static final IntegerProperty PROP_INDEX_LIMIT = new IntegerProperty("autosave.index-limit", 1000);
103    /**
104     * Defines if a notification should be displayed after each autosave
105     */
106    public static final BooleanProperty PROP_NOTIFICATION = new BooleanProperty("autosave.notification", false);
107
108    protected static final class AutosaveLayerInfo<T extends AbstractModifiableLayer> {
109        private final T layer;
110        private String layerName;
111        private String layerFileName;
112        private final Deque<File> backupFiles = new LinkedList<>();
113
114        AutosaveLayerInfo(T layer) {
115            this.layer = layer;
116        }
117    }
118
119    private final DataSetListenerAdapter datasetAdapter = new DataSetListenerAdapter(this);
120    private final Set<Data> changedData = new HashSet<>();
121    private final List<AutosaveLayerInfo<?>> layersInfo = new ArrayList<>();
122    private final Object layersLock = new Object();
123    private final Deque<File> deletedLayers = new LinkedList<>();
124
125    private final File autosaveDir = new File(Config.getDirs().getUserDataDirectory(true), AUTOSAVE_DIR);
126    private final File deletedLayersDir = new File(Config.getDirs().getUserDataDirectory(true), DELETED_LAYERS_DIR);
127
128    /**
129     * Replies the autosave directory.
130     * @return the autosave directory
131     * @since 10299
132     */
133    public final Path getAutosaveDir() {
134        return autosaveDir.toPath();
135    }
136
137    /**
138     * Starts the autosave background task.
139     */
140    public void schedule() {
141        if (PROP_INTERVAL.get() > 0) {
142
143            if (!autosaveDir.exists() && !autosaveDir.mkdirs()) {
144                Logging.warn(tr("Unable to create directory {0}, autosave will be disabled", autosaveDir.getAbsolutePath()));
145                return;
146            }
147            if (!deletedLayersDir.exists() && !deletedLayersDir.mkdirs()) {
148                Logging.warn(tr("Unable to create directory {0}, autosave will be disabled", deletedLayersDir.getAbsolutePath()));
149                return;
150            }
151
152            File[] files = deletedLayersDir.listFiles();
153            if (files != null) {
154                try {
155                    Arrays.sort(files, Comparator.comparingLong(File::lastModified));
156                } catch (Exception e) {
157                    Logging.error(e);
158                }
159                deletedLayers.addAll(Arrays.asList(files));
160            }
161
162            new Timer(true).schedule(this, TimeUnit.SECONDS.toMillis(1), TimeUnit.SECONDS.toMillis(PROP_INTERVAL.get()));
163            MainApplication.getLayerManager().addAndFireLayerChangeListener(this);
164        }
165    }
166
167    private static String getFileName(String layerName, int index) {
168        String result = layerName;
169        for (char illegalCharacter : ILLEGAL_CHARACTERS) {
170            result = result.replaceAll(Pattern.quote(String.valueOf(illegalCharacter)),
171                    '&' + String.valueOf((int) illegalCharacter) + ';');
172        }
173        if (index != 0) {
174            result = result + '_' + index;
175        }
176        return result;
177    }
178
179    private void setLayerFileName(AutosaveLayerInfo<?> layer) {
180        int index = 0;
181        while (true) {
182            String filename = getFileName(layer.layer.getName(), index);
183            boolean foundTheSame = layersInfo.stream().anyMatch(info -> info != layer && filename.equals(info.layerFileName));
184            if (!foundTheSame) {
185                layer.layerFileName = filename;
186                return;
187            }
188
189            index++;
190        }
191    }
192
193    protected File getNewLayerFile(AutosaveLayerInfo<?> layer, Instant now, int startIndex) {
194        int index = startIndex;
195        while (true) {
196            String filename = String.format(Locale.ENGLISH, "%1$s_%2$tY%2$tm%2$td_%2$tH%2$tM%2$tS%2$tL%3$s",
197                    layer.layerFileName, Date.from(now), index == 0 ? "" : ('_' + Integer.toString(index)));
198            File result = new File(autosaveDir, filename + '.' +
199                    (layer.layer instanceof NoteLayer ?
200                            Config.getPref().get("autosave.notes.extension", "osn") :
201                            Config.getPref().get("autosave.extension", "osm")));
202            try {
203                if (index > PROP_INDEX_LIMIT.get())
204                    throw new IOException("index limit exceeded");
205                if (result.createNewFile()) {
206                    createNewPidFile(autosaveDir, filename);
207                    return result;
208                } else {
209                    Logging.warn(tr("Unable to create file {0}, other filename will be used", result.getAbsolutePath()));
210                }
211            } catch (IOException e) {
212                Logging.log(Logging.LEVEL_ERROR, tr("IOError while creating file, autosave will be skipped: {0}", e.getMessage()), e);
213                return null;
214            }
215            index++;
216        }
217    }
218
219    private static void createNewPidFile(File autosaveDir, String filename) {
220        File pidFile = new File(autosaveDir, filename+".pid");
221        try {
222            final String content = ManagementFactory.getRuntimeMXBean().getName();
223            Files.write(pidFile.toPath(), Collections.singleton(content), StandardCharsets.UTF_8);
224        } catch (IOException | SecurityException t) {
225            Logging.error(t);
226        }
227    }
228
229    private void savelayer(AutosaveLayerInfo<?> info) {
230        if (!info.layer.getName().equals(info.layerName)) {
231            setLayerFileName(info);
232            info.layerName = info.layer.getName();
233        }
234        try {
235            Data data = info.layer.getData();
236            if (data != null && changedData.remove(data)) {
237                File file = getNewLayerFile(info, Instant.now(), 0);
238                if (file != null) {
239                    info.backupFiles.add(file);
240                    info.layer.autosave(file);
241                }
242            }
243        } catch (IOException e) {
244            Logging.error(e);
245        }
246        while (info.backupFiles.size() > PROP_FILES_PER_LAYER.get()) {
247            File oldFile = info.backupFiles.remove();
248            if (Utils.deleteFile(oldFile, marktr("Unable to delete old backup file {0}"))) {
249                Utils.deleteFile(getPidFile(oldFile), marktr("Unable to delete old backup file {0}"));
250            }
251        }
252    }
253
254    @Override
255    public void run() {
256        synchronized (layersLock) {
257            try {
258                for (AutosaveLayerInfo<?> info: layersInfo) {
259                    savelayer(info);
260                }
261                changedData.clear();
262                if (PROP_NOTIFICATION.get() && !layersInfo.isEmpty()) {
263                    GuiHelper.runInEDT(this::displayNotification);
264                }
265            } catch (RuntimeException t) { // NOPMD
266                // Don't let exception stop time thread
267                Logging.error("Autosave failed:");
268                Logging.error(t);
269            }
270        }
271    }
272
273    protected void displayNotification() {
274        new Notification(tr("Your work has been saved automatically."))
275        .setDuration(Notification.TIME_SHORT)
276        .show();
277    }
278
279    @Override
280    public void layerOrderChanged(LayerOrderChangeEvent e) {
281        // Do nothing
282    }
283
284    private void registerNewlayer(OsmDataLayer layer) {
285        synchronized (layersLock) {
286            layer.getDataSet().addDataSetListener(datasetAdapter);
287            layersInfo.add(new AutosaveLayerInfo<>(layer));
288        }
289    }
290
291    private void registerNewlayer(NoteLayer layer) {
292        synchronized (layersLock) {
293            layer.getNoteData().addNoteDataUpdateListener(this);
294            layersInfo.add(new AutosaveLayerInfo<>(layer));
295        }
296    }
297
298    @Override
299    public void layerAdded(LayerAddEvent e) {
300        Layer layer = e.getAddedLayer();
301        if (layer.isSavable()) {
302            if (layer instanceof OsmDataLayer) {
303                registerNewlayer((OsmDataLayer) layer);
304            } else if (layer instanceof NoteLayer) {
305                registerNewlayer((NoteLayer) layer);
306            } else if (layer instanceof AbstractModifiableLayer) {
307                synchronized (layersLock) {
308                    layersInfo.add(new AutosaveLayerInfo<>((AbstractModifiableLayer) layer));
309                }
310            } else {
311                Logging.debug("Unsupported savable layer type: {0}", layer.getClass().getSimpleName());
312            }
313        }
314    }
315
316    @Override
317    public void layerRemoving(LayerRemoveEvent e) {
318        if (e.getRemovedLayer() instanceof OsmDataLayer) {
319            synchronized (layersLock) {
320                OsmDataLayer osmLayer = (OsmDataLayer) e.getRemovedLayer();
321                osmLayer.getDataSet().removeDataSetListener(datasetAdapter);
322                cleanupLayer(osmLayer);
323            }
324        } else if (e.getRemovedLayer() instanceof NoteLayer) {
325            synchronized (layersLock) {
326                NoteLayer noteLayer = (NoteLayer) e.getRemovedLayer();
327                noteLayer.getNoteData().removeNoteDataUpdateListener(this);
328                cleanupLayer(noteLayer);
329            }
330        } else if (e.getRemovedLayer() instanceof AbstractModifiableLayer) {
331            synchronized (layersLock) {
332                cleanupLayer((AbstractModifiableLayer) e.getRemovedLayer());
333            }
334        }
335    }
336
337    private void cleanupLayer(AbstractModifiableLayer removedLayer) {
338        Iterator<AutosaveLayerInfo<?>> it = layersInfo.iterator();
339        while (it.hasNext()) {
340            AutosaveLayerInfo<?> info = it.next();
341            if (info.layer == removedLayer) {
342
343                savelayer(info);
344                File lastFile = info.backupFiles.pollLast();
345                if (lastFile != null) {
346                    moveToDeletedLayersFolder(lastFile);
347                }
348                for (File file: info.backupFiles) {
349                    if (Utils.deleteFile(file)) {
350                        Utils.deleteFile(getPidFile(file));
351                    }
352                }
353
354                it.remove();
355            }
356        }
357    }
358
359    @Override
360    public void processDatasetEvent(AbstractDatasetChangedEvent event) {
361        dataUpdated(event.getDataset());
362    }
363
364    @Override
365    public void noteDataUpdated(NoteData data) {
366        dataUpdated(data);
367    }
368
369    /**
370     * Indicate that data has changed, and it might be a good idea to autosave.
371     *
372     * @param data The data that has changed
373     * @return See {@link Set#add}
374     * @since 16548
375     */
376    public boolean dataUpdated(Data data) {
377        return changedData.add(data);
378    }
379
380    @Override
381    public void selectedNoteChanged(NoteData noteData) {
382        // Do nothing
383    }
384
385    protected File getPidFile(File osmFile) {
386        return new File(autosaveDir, osmFile.getName().replaceFirst("[.][^.]+$", ".pid"));
387    }
388
389    /**
390     * Replies the list of .osm files still present in autosave dir, that are not currently managed by another instance of JOSM.
391     * These files are hence unsaved layers from an old instance of JOSM that crashed and may be recovered by this instance.
392     * @return The list of .osm files still present in autosave dir, that are not currently managed by another instance of JOSM
393     */
394    public List<File> getUnsavedLayersFiles() {
395        List<File> result = new ArrayList<>();
396        try {
397            File[] files = autosaveDir.listFiles((FileFilter)
398                    pathname -> OsmImporter.FILE_FILTER.accept(pathname) || NoteImporter.FILE_FILTER.accept(pathname));
399            if (files == null)
400                return result;
401            for (File file: files) {
402                if (file.isFile()) {
403                    boolean skipFile = false;
404                    File pidFile = getPidFile(file);
405                    if (pidFile.exists()) {
406                        try (BufferedReader reader = Files.newBufferedReader(pidFile.toPath(), StandardCharsets.UTF_8)) {
407                            String jvmId = reader.readLine();
408                            if (jvmId != null) {
409                                String pid = jvmId.split("@", -1)[0];
410                                skipFile = jvmPerfDataFileExists(pid);
411                            }
412                        } catch (IOException | SecurityException t) {
413                            Logging.error(t);
414                        }
415                    }
416                    if (!skipFile) {
417                        result.add(file);
418                    }
419                }
420            }
421        } catch (SecurityException e) {
422            Logging.log(Logging.LEVEL_ERROR, "Unable to list unsaved layers files", e);
423        }
424        return result;
425    }
426
427    private static boolean jvmPerfDataFileExists(final String jvmId) {
428        File jvmDir = new File(getSystemProperty("java.io.tmpdir") + File.separator + "hsperfdata_" + getSystemProperty("user.name"));
429        if (jvmDir.exists() && jvmDir.canRead()) {
430            File[] files = jvmDir.listFiles((FileFilter) file -> file.getName().equals(jvmId) && file.isFile());
431            return files != null && files.length == 1;
432        }
433        return false;
434    }
435
436    /**
437     * Recover the unsaved layers and open them asynchronously.
438     * @return A future that can be used to wait for the completion of this task.
439     */
440    public Future<?> recoverUnsavedLayers() {
441        List<File> files = getUnsavedLayersFiles();
442        final OpenFileTask openFileTsk = new OpenFileTask(files, null, tr("Restoring files"));
443        final Future<?> openFilesFuture = MainApplication.worker.submit(openFileTsk);
444        return MainApplication.worker.submit(() -> {
445            try {
446                // Wait for opened tasks to be generated.
447                openFilesFuture.get();
448                for (File f: openFileTsk.getSuccessfullyOpenedFiles()) {
449                    moveToDeletedLayersFolder(f);
450                }
451            } catch (InterruptedException | ExecutionException e) {
452                Logging.error(e);
453            }
454        });
455    }
456
457    /**
458     * Move file to the deleted layers directory.
459     * If moving does not work, it will try to delete the file directly.
460     * Afterwards, if the number of deleted layers gets larger than PROP_DELETED_LAYERS,
461     * some files in the deleted layers directory will be removed.
462     *
463     * @param f the file, usually from the autosave dir
464     */
465    private void moveToDeletedLayersFolder(File f) {
466        File backupFile = new File(deletedLayersDir, f.getName());
467        File pidFile = getPidFile(f);
468
469        if (backupFile.exists()) {
470            deletedLayers.remove(backupFile);
471            Utils.deleteFile(backupFile, marktr("Unable to delete old backup file {0}"));
472        }
473        if (f.renameTo(backupFile)) {
474            deletedLayers.add(backupFile);
475            Utils.deleteFile(pidFile);
476        } else {
477            Logging.warn(String.format("Could not move autosaved file %s to %s folder", f.getName(), deletedLayersDir.getName()));
478            // we cannot move to deleted folder, so just try to delete it directly
479            if (Utils.deleteFile(f, marktr("Unable to delete backup file {0}"))) {
480                Utils.deleteFile(pidFile, marktr("Unable to delete PID file {0}"));
481            }
482        }
483        while (deletedLayers.size() > PROP_DELETED_LAYERS.get()) {
484            File next = deletedLayers.remove();
485            if (next == null) {
486                break;
487            }
488            Utils.deleteFile(next, marktr("Unable to delete archived backup file {0}"));
489        }
490    }
491
492    /**
493     * Mark all unsaved layers as deleted. They are still preserved in the deleted layers folder.
494     */
495    public void discardUnsavedLayers() {
496        for (File f: getUnsavedLayersFiles()) {
497            moveToDeletedLayersFolder(f);
498        }
499    }
500}