001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.actions.downloadtasks;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.net.MalformedURLException;
008import java.net.URL;
009import java.util.ArrayList;
010import java.util.Arrays;
011import java.util.Collection;
012import java.util.Collections;
013import java.util.HashSet;
014import java.util.Objects;
015import java.util.Optional;
016import java.util.Set;
017import java.util.concurrent.Future;
018import java.util.regex.Matcher;
019import java.util.regex.Pattern;
020import java.util.stream.Stream;
021
022import org.openstreetmap.josm.data.Bounds;
023import org.openstreetmap.josm.data.DataSource;
024import org.openstreetmap.josm.data.ProjectionBounds;
025import org.openstreetmap.josm.data.ViewportData;
026import org.openstreetmap.josm.data.coor.LatLon;
027import org.openstreetmap.josm.data.osm.DataSet;
028import org.openstreetmap.josm.data.osm.OsmPrimitive;
029import org.openstreetmap.josm.data.osm.Relation;
030import org.openstreetmap.josm.data.osm.Way;
031import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor;
032import org.openstreetmap.josm.gui.MainApplication;
033import org.openstreetmap.josm.gui.MapFrame;
034import org.openstreetmap.josm.gui.PleaseWaitRunnable;
035import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask;
036import org.openstreetmap.josm.gui.layer.OsmDataLayer;
037import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
038import org.openstreetmap.josm.gui.progress.ProgressMonitor;
039import org.openstreetmap.josm.io.BoundingBoxDownloader;
040import org.openstreetmap.josm.io.Compression;
041import org.openstreetmap.josm.io.OsmServerLocationReader;
042import org.openstreetmap.josm.io.OsmServerReader;
043import org.openstreetmap.josm.io.OsmTransferCanceledException;
044import org.openstreetmap.josm.io.OsmTransferException;
045import org.openstreetmap.josm.io.OverpassDownloadReader;
046import org.openstreetmap.josm.io.UrlPatterns.OsmUrlPattern;
047import org.openstreetmap.josm.spi.preferences.Config;
048import org.openstreetmap.josm.tools.Logging;
049import org.openstreetmap.josm.tools.Utils;
050import org.xml.sax.SAXException;
051
052/**
053 * Open the download dialog and download the data.
054 * Run in the worker thread.
055 */
056public class DownloadOsmTask extends AbstractDownloadTask<DataSet> {
057
058    protected Bounds currentBounds;
059    protected DownloadTask downloadTask;
060
061    protected String newLayerName;
062
063    /** This allows subclasses to ignore this warning */
064    protected boolean warnAboutEmptyArea = true;
065
066    protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data=";
067
068    private static final String NO_DATA_FOUND = tr("No data found in this area.");
069    static {
070        PostDownloadHandler.addNoDataErrorMessage(NO_DATA_FOUND);
071    }
072
073    @Override
074    public String[] getPatterns() {
075        if (this.getClass() == DownloadOsmTask.class) {
076            return patterns(OsmUrlPattern.class);
077        } else {
078            return super.getPatterns();
079        }
080    }
081
082    @Override
083    public String getTitle() {
084        if (this.getClass() == DownloadOsmTask.class) {
085            return tr("Download OSM");
086        } else {
087            return super.getTitle();
088        }
089    }
090
091    @Override
092    public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
093        return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor);
094    }
095
096    /**
097     * Asynchronously launches the download task for a given bounding box.
098     *
099     * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor.
100     * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to
101     * be discarded.
102     *
103     * You can wait for the asynchronous download task to finish by synchronizing on the returned
104     * {@link Future}, but make sure not to freeze up JOSM. Example:
105     * <pre>
106     *    Future&lt;?&gt; future = task.download(...);
107     *    // DON'T run this on the Swing EDT or JOSM will freeze
108     *    future.get(); // waits for the dowload task to complete
109     * </pre>
110     *
111     * The following example uses a pattern which is better suited if a task is launched from
112     * the Swing EDT:
113     * <pre>
114     *    final Future&lt;?&gt; future = task.download(...);
115     *    Runnable runAfterTask = new Runnable() {
116     *       public void run() {
117     *           // this is not strictly necessary because of the type of executor service
118     *           // Main.worker is initialized with, but it doesn't harm either
119     *           //
120     *           future.get(); // wait for the download task to complete
121     *           doSomethingAfterTheTaskCompleted();
122     *       }
123     *    }
124     *    MainApplication.worker.submit(runAfterTask);
125     * </pre>
126     * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm})
127     * @param settings download settings
128     * @param downloadArea the area to download
129     * @param progressMonitor the progressMonitor
130     * @return the future representing the asynchronous task
131     * @since 13927
132     */
133    public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
134        return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea);
135    }
136
137    protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) {
138        this.downloadTask = downloadTask;
139        this.currentBounds = new Bounds(downloadArea);
140        // We need submit instead of execute so we can wait for it to finish and get the error
141        // message if necessary. If no one calls getErrorMessage() it just behaves like execute.
142        return MainApplication.worker.submit(downloadTask);
143    }
144
145    /**
146     * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed.
147     * @param url the original URL
148     * @return the modified URL
149     */
150    protected String modifyUrlBeforeLoad(String url) {
151        return url;
152    }
153
154    /**
155     * Loads a given URL from the OSM Server
156     * @param settings download settings
157     * @param url The URL as String
158     */
159    @Override
160    public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) {
161        String newUrl = modifyUrlBeforeLoad(url);
162        Optional<OsmUrlPattern> urlPattern = Arrays.stream(OsmUrlPattern.values()).filter(p -> p.matches(newUrl)).findFirst();
163        downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor, true, Compression.byExtension(newUrl));
164        currentBounds = null;
165        // Extract .osm filename from URL to set the new layer name
166        extractOsmFilename(settings, urlPattern.orElse(OsmUrlPattern.EXTERNAL_OSM_FILE).pattern(), newUrl);
167        return MainApplication.worker.submit(downloadTask);
168    }
169
170    protected OsmServerReader getOsmServerReader(String url) {
171        try {
172            String host = new URL(url).getHost();
173            for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) {
174                if (host.equals(new URL(knownOverpassServer).getHost())) {
175                    int index = url.indexOf(OVERPASS_INTERPRETER_DATA);
176                    if (index > 0) {
177                        return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer,
178                                Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length())));
179                    }
180                }
181            }
182        } catch (MalformedURLException e) {
183            Logging.error(e);
184        }
185        return new OsmServerLocationReader(url);
186    }
187
188    protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) {
189        newLayerName = settings.getLayerName();
190        if (Utils.isEmpty(newLayerName)) {
191            Matcher matcher = Pattern.compile(pattern).matcher(url);
192            newLayerName = matcher.matches() && matcher.groupCount() > 0 ? Utils.decodeUrl(matcher.group(1)) : null;
193        }
194    }
195
196    @Override
197    public void cancel() {
198        if (downloadTask != null) {
199            downloadTask.cancel();
200        }
201    }
202
203    @Override
204    public boolean isSafeForRemotecontrolRequests() {
205        return true;
206    }
207
208    @Override
209    public ProjectionBounds getDownloadProjectionBounds() {
210        return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null;
211    }
212
213    protected Collection<OsmPrimitive> searchPotentiallyDeletedPrimitives(DataSet ds) {
214        return downloadTask.searchPrimitivesToUpdate(currentBounds, ds);
215    }
216
217    protected final void rememberDownloadedBounds(Bounds bounds) {
218        if (bounds != null) {
219            Config.getPref().put("osm-download.bounds", bounds.encodeAsString(";"));
220        }
221    }
222
223    /**
224     * Superclass of internal download task.
225     * @since 7636
226     */
227    public abstract static class AbstractInternalTask extends PleaseWaitRunnable {
228
229        protected final DownloadParams settings;
230        protected final boolean zoomAfterDownload;
231        protected DataSet dataSet;
232
233        /**
234         * Constructs a new {@code AbstractInternalTask}.
235         * @param settings download settings
236         * @param title message for the user
237         * @param ignoreException If true, exception will be propagated to calling code. If false then
238         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
239         * then use false unless you read result of task (because exception will get lost if you don't)
240         * @param zoomAfterDownload If true, the map view will zoom to download area after download
241         */
242        protected AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) {
243            super(title, ignoreException);
244            this.settings = Objects.requireNonNull(settings);
245            this.zoomAfterDownload = zoomAfterDownload;
246        }
247
248        /**
249         * Constructs a new {@code AbstractInternalTask}.
250         * @param settings download settings
251         * @param title message for the user
252         * @param progressMonitor progress monitor
253         * @param ignoreException If true, exception will be propagated to calling code. If false then
254         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
255         * then use false unless you read result of task (because exception will get lost if you don't)
256         * @param zoomAfterDownload If true, the map view will zoom to download area after download
257         */
258        protected AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException,
259                boolean zoomAfterDownload) {
260            super(title, progressMonitor, ignoreException);
261            this.settings = Objects.requireNonNull(settings);
262            this.zoomAfterDownload = zoomAfterDownload;
263        }
264
265        protected OsmDataLayer getEditLayer() {
266            return MainApplication.getLayerManager().getEditLayer();
267        }
268
269        private static Stream<OsmDataLayer> getModifiableDataLayers() {
270            return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class)
271                    .stream().filter(OsmDataLayer::isDownloadable);
272        }
273
274        /**
275         * Returns the number of modifiable data layers
276         * @return number of modifiable data layers
277         * @since 13434
278         */
279        protected long getNumModifiableDataLayers() {
280            return getModifiableDataLayers().count();
281        }
282
283        /**
284         * Returns the first modifiable data layer
285         * @return the first modifiable data layer
286         * @since 13434
287         */
288        protected OsmDataLayer getFirstModifiableDataLayer() {
289            return getModifiableDataLayers().findFirst().orElse(null);
290        }
291
292        /**
293         * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or
294         * {@link OsmDataLayer#createNewName()} if the former option is {@code null}.
295         *
296         * @return a name for a new layer
297         * @since 14347
298         */
299        protected String generateLayerName() {
300            return Optional.ofNullable(settings.getLayerName())
301                .filter(layerName -> !Utils.isStripEmpty(layerName))
302                .orElse(OsmDataLayer.createNewName());
303        }
304
305        /**
306         * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed.
307         * If you want to change how the name is determined, consider overriding
308         * {@link #generateLayerName()} instead.
309         *
310         * @param ds the dataset on which the layer is based, must be non-null
311         * @param layerName the name of the new layer, must be either non-blank or non-present
312         * @return a new instance of {@link OsmDataLayer} constructed with the given arguments
313         * @since 14347
314         */
315        protected OsmDataLayer createNewLayer(final DataSet ds, final Optional<String> layerName) {
316            if (layerName.filter(Utils::isStripEmpty).isPresent()) {
317                throw new IllegalArgumentException("Blank layer name!");
318            }
319            return new OsmDataLayer(
320                Objects.requireNonNull(ds, "dataset parameter"),
321                layerName.orElseGet(this::generateLayerName),
322                null
323            );
324        }
325
326        /**
327         * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset
328         * from field {@link #dataSet} and applies the settings from field {@link #settings}.
329         *
330         * @param layerName an optional layer name, must be non-blank if the [Optional] is present
331         * @return a newly constructed layer
332         * @since 14347
333         */
334        protected final OsmDataLayer createNewLayer(final Optional<String> layerName) {
335            Optional.ofNullable(settings.getDownloadPolicy())
336                .ifPresent(dataSet::setDownloadPolicy);
337            Optional.ofNullable(settings.getUploadPolicy())
338                .ifPresent(dataSet::setUploadPolicy);
339            if (dataSet.isLocked() && !settings.isLocked()) {
340                dataSet.unlock();
341            } else if (!dataSet.isLocked() && settings.isLocked()) {
342                dataSet.lock();
343            }
344            return createNewLayer(dataSet, layerName);
345        }
346
347        protected Optional<ProjectionBounds> computeBbox(Bounds bounds) {
348            BoundingXYVisitor v = new BoundingXYVisitor();
349            if (bounds != null) {
350                v.visit(bounds);
351            } else {
352                v.computeBoundingBox(dataSet.getNodes());
353            }
354            return Optional.ofNullable(v.getBounds());
355        }
356
357        protected OsmDataLayer addNewLayerIfRequired(String newLayerName) {
358            long numDataLayers = getNumModifiableDataLayers();
359            if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) {
360                // the user explicitly wants a new layer, we don't have any layer at all
361                // or it is not clear which layer to merge to
362                final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(it -> !Utils.isStripEmpty(it)));
363                MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload);
364                return layer;
365            }
366            return null;
367        }
368
369        protected void loadData(String newLayerName, Bounds bounds) {
370            OsmDataLayer layer = addNewLayerIfRequired(newLayerName);
371            if (layer == null) {
372                layer = getEditLayer();
373                if (layer == null || !layer.isDownloadable()) {
374                    layer = getFirstModifiableDataLayer();
375                }
376                Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet());
377                layer.mergeFrom(dataSet);
378                MapFrame map = MainApplication.getMap();
379                if (map != null && zoomAfterDownload) {
380                    computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo);
381                }
382                if (!primitivesToUpdate.isEmpty()) {
383                    MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate));
384                }
385            }
386            layer.onPostDownloadFromServer(); // for existing and newly added layer, see #19816
387        }
388
389        /**
390         * Look for primitives deleted on server (thus absent from downloaded data)
391         * but still present in existing data layer
392         * @param bounds download bounds
393         * @param ds existing data set
394         * @return the primitives to update
395         */
396        protected Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) {
397            if (bounds == null)
398                return Collections.emptySet();
399            Collection<OsmPrimitive> col = new ArrayList<>();
400            ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add);
401            if (!col.isEmpty()) {
402                Set<Way> ways = new HashSet<>();
403                Set<Relation> rels = new HashSet<>();
404                for (OsmPrimitive n : col) {
405                    for (OsmPrimitive ref : n.getReferrers()) {
406                        if (ref.isNew()) {
407                            continue;
408                        } else if (ref instanceof Way) {
409                            ways.add((Way) ref);
410                        } else if (ref instanceof Relation) {
411                            rels.add((Relation) ref);
412                        }
413                    }
414                }
415                ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add);
416                rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add);
417            }
418            return Collections.unmodifiableCollection(col);
419        }
420    }
421
422    protected class DownloadTask extends AbstractInternalTask {
423        protected final OsmServerReader reader;
424        protected final Compression compression;
425
426        /**
427         * Constructs a new {@code DownloadTask}.
428         * @param settings download settings
429         * @param reader OSM data reader
430         * @param progressMonitor progress monitor
431         * @since 13927
432         */
433        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) {
434            this(settings, reader, progressMonitor, true);
435        }
436
437        /**
438         * Constructs a new {@code DownloadTask}.
439         * @param settings download settings
440         * @param reader OSM data reader
441         * @param progressMonitor progress monitor
442         * @param zoomAfterDownload If true, the map view will zoom to download area after download
443         * @since 13927
444         */
445        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) {
446            this(settings, reader, progressMonitor, zoomAfterDownload, Compression.NONE);
447        }
448
449        /**
450         * Constructs a new {@code DownloadTask}.
451         * @param settings download settings
452         * @param reader OSM data reader
453         * @param progressMonitor progress monitor
454         * @param zoomAfterDownload If true, the map view will zoom to download area after download
455         * @param compression compression to use
456         * @since 15784
457         */
458        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload,
459                Compression compression) {
460            super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload);
461            this.reader = Objects.requireNonNull(reader);
462            this.compression = compression;
463        }
464
465        protected DataSet parseDataSet() throws OsmTransferException {
466            ProgressMonitor subTaskMonitor = progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false);
467            // Don't call parseOsm signature with compression if not needed, too many implementations to update before to avoid side effects
468            return compression != null && compression != Compression.NONE ?
469                    reader.parseOsm(subTaskMonitor, compression) : reader.parseOsm(subTaskMonitor);
470        }
471
472        @Override
473        public void realRun() throws IOException, SAXException, OsmTransferException {
474            try {
475                if (isCanceled())
476                    return;
477                dataSet = parseDataSet();
478            } catch (OsmTransferException e) {
479                if (isCanceled()) {
480                    Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString()));
481                    return;
482                }
483                if (e instanceof OsmTransferCanceledException) {
484                    setCanceled(true);
485                    return;
486                } else {
487                    rememberException(e);
488                }
489                DownloadOsmTask.this.setFailed(true);
490            }
491        }
492
493        @Override
494        protected void finish() {
495            if (isFailed() || isCanceled())
496                return;
497            if (dataSet == null)
498                return; // user canceled download or error occurred
499            if (dataSet.allPrimitives().isEmpty()) {
500                if (warnAboutEmptyArea) {
501                    rememberErrorMessage(NO_DATA_FOUND);
502                }
503                String remark = dataSet.getRemark();
504                if (!Utils.isEmpty(remark)) {
505                    rememberErrorMessage(remark);
506                }
507                if (!(reader instanceof BoundingBoxDownloader)
508                        || ((BoundingBoxDownloader) reader).considerAsFullDownload()) {
509                    // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work
510                    dataSet.addDataSource(new DataSource(
511                            currentBounds != null ? currentBounds : new Bounds(LatLon.ZERO), "OpenStreetMap server"));
512                }
513            }
514
515            rememberDownloadedBounds(currentBounds);
516            rememberDownloadedData(dataSet);
517            loadData(newLayerName, currentBounds);
518        }
519
520        @Override
521        protected void cancel() {
522            setCanceled(true);
523            if (reader != null) {
524                reader.cancel();
525            }
526        }
527    }
528
529    @Override
530    public String getConfirmationMessage(URL url) {
531        if (OsmUrlPattern.OSM_API_URL.matches(url)) {
532            Collection<String> items = new ArrayList<>();
533            items.add(tr("OSM Server URL:") + ' ' + url.getHost());
534            items.add(tr("Command")+": "+url.getPath());
535            if (url.getQuery() != null) {
536                items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", ")));
537            }
538            return Utils.joinAsHtmlUnorderedList(items);
539        }
540        return null;
541    }
542}