001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.actions.downloadtasks; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.net.MalformedURLException; 008import java.net.URL; 009import java.util.ArrayList; 010import java.util.Arrays; 011import java.util.Collection; 012import java.util.Collections; 013import java.util.HashSet; 014import java.util.Objects; 015import java.util.Optional; 016import java.util.Set; 017import java.util.concurrent.Future; 018import java.util.regex.Matcher; 019import java.util.regex.Pattern; 020import java.util.stream.Stream; 021 022import org.openstreetmap.josm.data.Bounds; 023import org.openstreetmap.josm.data.DataSource; 024import org.openstreetmap.josm.data.ProjectionBounds; 025import org.openstreetmap.josm.data.ViewportData; 026import org.openstreetmap.josm.data.coor.LatLon; 027import org.openstreetmap.josm.data.osm.DataSet; 028import org.openstreetmap.josm.data.osm.OsmPrimitive; 029import org.openstreetmap.josm.data.osm.Relation; 030import org.openstreetmap.josm.data.osm.Way; 031import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; 032import org.openstreetmap.josm.gui.MainApplication; 033import org.openstreetmap.josm.gui.MapFrame; 034import org.openstreetmap.josm.gui.PleaseWaitRunnable; 035import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask; 036import org.openstreetmap.josm.gui.layer.OsmDataLayer; 037import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 038import org.openstreetmap.josm.gui.progress.ProgressMonitor; 039import org.openstreetmap.josm.io.BoundingBoxDownloader; 040import org.openstreetmap.josm.io.Compression; 041import org.openstreetmap.josm.io.OsmServerLocationReader; 042import org.openstreetmap.josm.io.OsmServerReader; 043import org.openstreetmap.josm.io.OsmTransferCanceledException; 044import org.openstreetmap.josm.io.OsmTransferException; 045import org.openstreetmap.josm.io.OverpassDownloadReader; 046import org.openstreetmap.josm.io.UrlPatterns.OsmUrlPattern; 047import org.openstreetmap.josm.spi.preferences.Config; 048import org.openstreetmap.josm.tools.Logging; 049import org.openstreetmap.josm.tools.Utils; 050import org.xml.sax.SAXException; 051 052/** 053 * Open the download dialog and download the data. 054 * Run in the worker thread. 055 */ 056public class DownloadOsmTask extends AbstractDownloadTask<DataSet> { 057 058 protected Bounds currentBounds; 059 protected DownloadTask downloadTask; 060 061 protected String newLayerName; 062 063 /** This allows subclasses to ignore this warning */ 064 protected boolean warnAboutEmptyArea = true; 065 066 protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data="; 067 068 private static final String NO_DATA_FOUND = tr("No data found in this area."); 069 static { 070 PostDownloadHandler.addNoDataErrorMessage(NO_DATA_FOUND); 071 } 072 073 @Override 074 public String[] getPatterns() { 075 if (this.getClass() == DownloadOsmTask.class) { 076 return patterns(OsmUrlPattern.class); 077 } else { 078 return super.getPatterns(); 079 } 080 } 081 082 @Override 083 public String getTitle() { 084 if (this.getClass() == DownloadOsmTask.class) { 085 return tr("Download OSM"); 086 } else { 087 return super.getTitle(); 088 } 089 } 090 091 @Override 092 public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 093 return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor); 094 } 095 096 /** 097 * Asynchronously launches the download task for a given bounding box. 098 * 099 * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor. 100 * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to 101 * be discarded. 102 * 103 * You can wait for the asynchronous download task to finish by synchronizing on the returned 104 * {@link Future}, but make sure not to freeze up JOSM. Example: 105 * <pre> 106 * Future<?> future = task.download(...); 107 * // DON'T run this on the Swing EDT or JOSM will freeze 108 * future.get(); // waits for the dowload task to complete 109 * </pre> 110 * 111 * The following example uses a pattern which is better suited if a task is launched from 112 * the Swing EDT: 113 * <pre> 114 * final Future<?> future = task.download(...); 115 * Runnable runAfterTask = new Runnable() { 116 * public void run() { 117 * // this is not strictly necessary because of the type of executor service 118 * // Main.worker is initialized with, but it doesn't harm either 119 * // 120 * future.get(); // wait for the download task to complete 121 * doSomethingAfterTheTaskCompleted(); 122 * } 123 * } 124 * MainApplication.worker.submit(runAfterTask); 125 * </pre> 126 * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm}) 127 * @param settings download settings 128 * @param downloadArea the area to download 129 * @param progressMonitor the progressMonitor 130 * @return the future representing the asynchronous task 131 * @since 13927 132 */ 133 public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 134 return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea); 135 } 136 137 protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) { 138 this.downloadTask = downloadTask; 139 this.currentBounds = new Bounds(downloadArea); 140 // We need submit instead of execute so we can wait for it to finish and get the error 141 // message if necessary. If no one calls getErrorMessage() it just behaves like execute. 142 return MainApplication.worker.submit(downloadTask); 143 } 144 145 /** 146 * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed. 147 * @param url the original URL 148 * @return the modified URL 149 */ 150 protected String modifyUrlBeforeLoad(String url) { 151 return url; 152 } 153 154 /** 155 * Loads a given URL from the OSM Server 156 * @param settings download settings 157 * @param url The URL as String 158 */ 159 @Override 160 public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) { 161 String newUrl = modifyUrlBeforeLoad(url); 162 Optional<OsmUrlPattern> urlPattern = Arrays.stream(OsmUrlPattern.values()).filter(p -> p.matches(newUrl)).findFirst(); 163 downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor, true, Compression.byExtension(newUrl)); 164 currentBounds = null; 165 // Extract .osm filename from URL to set the new layer name 166 extractOsmFilename(settings, urlPattern.orElse(OsmUrlPattern.EXTERNAL_OSM_FILE).pattern(), newUrl); 167 return MainApplication.worker.submit(downloadTask); 168 } 169 170 protected OsmServerReader getOsmServerReader(String url) { 171 try { 172 String host = new URL(url).getHost(); 173 for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) { 174 if (host.equals(new URL(knownOverpassServer).getHost())) { 175 int index = url.indexOf(OVERPASS_INTERPRETER_DATA); 176 if (index > 0) { 177 return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer, 178 Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length()))); 179 } 180 } 181 } 182 } catch (MalformedURLException e) { 183 Logging.error(e); 184 } 185 return new OsmServerLocationReader(url); 186 } 187 188 protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) { 189 newLayerName = settings.getLayerName(); 190 if (Utils.isEmpty(newLayerName)) { 191 Matcher matcher = Pattern.compile(pattern).matcher(url); 192 newLayerName = matcher.matches() && matcher.groupCount() > 0 ? Utils.decodeUrl(matcher.group(1)) : null; 193 } 194 } 195 196 @Override 197 public void cancel() { 198 if (downloadTask != null) { 199 downloadTask.cancel(); 200 } 201 } 202 203 @Override 204 public boolean isSafeForRemotecontrolRequests() { 205 return true; 206 } 207 208 @Override 209 public ProjectionBounds getDownloadProjectionBounds() { 210 return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null; 211 } 212 213 protected Collection<OsmPrimitive> searchPotentiallyDeletedPrimitives(DataSet ds) { 214 return downloadTask.searchPrimitivesToUpdate(currentBounds, ds); 215 } 216 217 protected final void rememberDownloadedBounds(Bounds bounds) { 218 if (bounds != null) { 219 Config.getPref().put("osm-download.bounds", bounds.encodeAsString(";")); 220 } 221 } 222 223 /** 224 * Superclass of internal download task. 225 * @since 7636 226 */ 227 public abstract static class AbstractInternalTask extends PleaseWaitRunnable { 228 229 protected final DownloadParams settings; 230 protected final boolean zoomAfterDownload; 231 protected DataSet dataSet; 232 233 /** 234 * Constructs a new {@code AbstractInternalTask}. 235 * @param settings download settings 236 * @param title message for the user 237 * @param ignoreException If true, exception will be propagated to calling code. If false then 238 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 239 * then use false unless you read result of task (because exception will get lost if you don't) 240 * @param zoomAfterDownload If true, the map view will zoom to download area after download 241 */ 242 protected AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) { 243 super(title, ignoreException); 244 this.settings = Objects.requireNonNull(settings); 245 this.zoomAfterDownload = zoomAfterDownload; 246 } 247 248 /** 249 * Constructs a new {@code AbstractInternalTask}. 250 * @param settings download settings 251 * @param title message for the user 252 * @param progressMonitor progress monitor 253 * @param ignoreException If true, exception will be propagated to calling code. If false then 254 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 255 * then use false unless you read result of task (because exception will get lost if you don't) 256 * @param zoomAfterDownload If true, the map view will zoom to download area after download 257 */ 258 protected AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException, 259 boolean zoomAfterDownload) { 260 super(title, progressMonitor, ignoreException); 261 this.settings = Objects.requireNonNull(settings); 262 this.zoomAfterDownload = zoomAfterDownload; 263 } 264 265 protected OsmDataLayer getEditLayer() { 266 return MainApplication.getLayerManager().getEditLayer(); 267 } 268 269 private static Stream<OsmDataLayer> getModifiableDataLayers() { 270 return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class) 271 .stream().filter(OsmDataLayer::isDownloadable); 272 } 273 274 /** 275 * Returns the number of modifiable data layers 276 * @return number of modifiable data layers 277 * @since 13434 278 */ 279 protected long getNumModifiableDataLayers() { 280 return getModifiableDataLayers().count(); 281 } 282 283 /** 284 * Returns the first modifiable data layer 285 * @return the first modifiable data layer 286 * @since 13434 287 */ 288 protected OsmDataLayer getFirstModifiableDataLayer() { 289 return getModifiableDataLayers().findFirst().orElse(null); 290 } 291 292 /** 293 * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or 294 * {@link OsmDataLayer#createNewName()} if the former option is {@code null}. 295 * 296 * @return a name for a new layer 297 * @since 14347 298 */ 299 protected String generateLayerName() { 300 return Optional.ofNullable(settings.getLayerName()) 301 .filter(layerName -> !Utils.isStripEmpty(layerName)) 302 .orElse(OsmDataLayer.createNewName()); 303 } 304 305 /** 306 * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed. 307 * If you want to change how the name is determined, consider overriding 308 * {@link #generateLayerName()} instead. 309 * 310 * @param ds the dataset on which the layer is based, must be non-null 311 * @param layerName the name of the new layer, must be either non-blank or non-present 312 * @return a new instance of {@link OsmDataLayer} constructed with the given arguments 313 * @since 14347 314 */ 315 protected OsmDataLayer createNewLayer(final DataSet ds, final Optional<String> layerName) { 316 if (layerName.filter(Utils::isStripEmpty).isPresent()) { 317 throw new IllegalArgumentException("Blank layer name!"); 318 } 319 return new OsmDataLayer( 320 Objects.requireNonNull(ds, "dataset parameter"), 321 layerName.orElseGet(this::generateLayerName), 322 null 323 ); 324 } 325 326 /** 327 * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset 328 * from field {@link #dataSet} and applies the settings from field {@link #settings}. 329 * 330 * @param layerName an optional layer name, must be non-blank if the [Optional] is present 331 * @return a newly constructed layer 332 * @since 14347 333 */ 334 protected final OsmDataLayer createNewLayer(final Optional<String> layerName) { 335 Optional.ofNullable(settings.getDownloadPolicy()) 336 .ifPresent(dataSet::setDownloadPolicy); 337 Optional.ofNullable(settings.getUploadPolicy()) 338 .ifPresent(dataSet::setUploadPolicy); 339 if (dataSet.isLocked() && !settings.isLocked()) { 340 dataSet.unlock(); 341 } else if (!dataSet.isLocked() && settings.isLocked()) { 342 dataSet.lock(); 343 } 344 return createNewLayer(dataSet, layerName); 345 } 346 347 protected Optional<ProjectionBounds> computeBbox(Bounds bounds) { 348 BoundingXYVisitor v = new BoundingXYVisitor(); 349 if (bounds != null) { 350 v.visit(bounds); 351 } else { 352 v.computeBoundingBox(dataSet.getNodes()); 353 } 354 return Optional.ofNullable(v.getBounds()); 355 } 356 357 protected OsmDataLayer addNewLayerIfRequired(String newLayerName) { 358 long numDataLayers = getNumModifiableDataLayers(); 359 if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) { 360 // the user explicitly wants a new layer, we don't have any layer at all 361 // or it is not clear which layer to merge to 362 final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(it -> !Utils.isStripEmpty(it))); 363 MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload); 364 return layer; 365 } 366 return null; 367 } 368 369 protected void loadData(String newLayerName, Bounds bounds) { 370 OsmDataLayer layer = addNewLayerIfRequired(newLayerName); 371 if (layer == null) { 372 layer = getEditLayer(); 373 if (layer == null || !layer.isDownloadable()) { 374 layer = getFirstModifiableDataLayer(); 375 } 376 Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet()); 377 layer.mergeFrom(dataSet); 378 MapFrame map = MainApplication.getMap(); 379 if (map != null && zoomAfterDownload) { 380 computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo); 381 } 382 if (!primitivesToUpdate.isEmpty()) { 383 MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate)); 384 } 385 } 386 layer.onPostDownloadFromServer(); // for existing and newly added layer, see #19816 387 } 388 389 /** 390 * Look for primitives deleted on server (thus absent from downloaded data) 391 * but still present in existing data layer 392 * @param bounds download bounds 393 * @param ds existing data set 394 * @return the primitives to update 395 */ 396 protected Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) { 397 if (bounds == null) 398 return Collections.emptySet(); 399 Collection<OsmPrimitive> col = new ArrayList<>(); 400 ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add); 401 if (!col.isEmpty()) { 402 Set<Way> ways = new HashSet<>(); 403 Set<Relation> rels = new HashSet<>(); 404 for (OsmPrimitive n : col) { 405 for (OsmPrimitive ref : n.getReferrers()) { 406 if (ref.isNew()) { 407 continue; 408 } else if (ref instanceof Way) { 409 ways.add((Way) ref); 410 } else if (ref instanceof Relation) { 411 rels.add((Relation) ref); 412 } 413 } 414 } 415 ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add); 416 rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add); 417 } 418 return Collections.unmodifiableCollection(col); 419 } 420 } 421 422 protected class DownloadTask extends AbstractInternalTask { 423 protected final OsmServerReader reader; 424 protected final Compression compression; 425 426 /** 427 * Constructs a new {@code DownloadTask}. 428 * @param settings download settings 429 * @param reader OSM data reader 430 * @param progressMonitor progress monitor 431 * @since 13927 432 */ 433 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) { 434 this(settings, reader, progressMonitor, true); 435 } 436 437 /** 438 * Constructs a new {@code DownloadTask}. 439 * @param settings download settings 440 * @param reader OSM data reader 441 * @param progressMonitor progress monitor 442 * @param zoomAfterDownload If true, the map view will zoom to download area after download 443 * @since 13927 444 */ 445 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) { 446 this(settings, reader, progressMonitor, zoomAfterDownload, Compression.NONE); 447 } 448 449 /** 450 * Constructs a new {@code DownloadTask}. 451 * @param settings download settings 452 * @param reader OSM data reader 453 * @param progressMonitor progress monitor 454 * @param zoomAfterDownload If true, the map view will zoom to download area after download 455 * @param compression compression to use 456 * @since 15784 457 */ 458 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload, 459 Compression compression) { 460 super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload); 461 this.reader = Objects.requireNonNull(reader); 462 this.compression = compression; 463 } 464 465 protected DataSet parseDataSet() throws OsmTransferException { 466 ProgressMonitor subTaskMonitor = progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false); 467 // Don't call parseOsm signature with compression if not needed, too many implementations to update before to avoid side effects 468 return compression != null && compression != Compression.NONE ? 469 reader.parseOsm(subTaskMonitor, compression) : reader.parseOsm(subTaskMonitor); 470 } 471 472 @Override 473 public void realRun() throws IOException, SAXException, OsmTransferException { 474 try { 475 if (isCanceled()) 476 return; 477 dataSet = parseDataSet(); 478 } catch (OsmTransferException e) { 479 if (isCanceled()) { 480 Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString())); 481 return; 482 } 483 if (e instanceof OsmTransferCanceledException) { 484 setCanceled(true); 485 return; 486 } else { 487 rememberException(e); 488 } 489 DownloadOsmTask.this.setFailed(true); 490 } 491 } 492 493 @Override 494 protected void finish() { 495 if (isFailed() || isCanceled()) 496 return; 497 if (dataSet == null) 498 return; // user canceled download or error occurred 499 if (dataSet.allPrimitives().isEmpty()) { 500 if (warnAboutEmptyArea) { 501 rememberErrorMessage(NO_DATA_FOUND); 502 } 503 String remark = dataSet.getRemark(); 504 if (!Utils.isEmpty(remark)) { 505 rememberErrorMessage(remark); 506 } 507 if (!(reader instanceof BoundingBoxDownloader) 508 || ((BoundingBoxDownloader) reader).considerAsFullDownload()) { 509 // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work 510 dataSet.addDataSource(new DataSource( 511 currentBounds != null ? currentBounds : new Bounds(LatLon.ZERO), "OpenStreetMap server")); 512 } 513 } 514 515 rememberDownloadedBounds(currentBounds); 516 rememberDownloadedData(dataSet); 517 loadData(newLayerName, currentBounds); 518 } 519 520 @Override 521 protected void cancel() { 522 setCanceled(true); 523 if (reader != null) { 524 reader.cancel(); 525 } 526 } 527 } 528 529 @Override 530 public String getConfirmationMessage(URL url) { 531 if (OsmUrlPattern.OSM_API_URL.matches(url)) { 532 Collection<String> items = new ArrayList<>(); 533 items.add(tr("OSM Server URL:") + ' ' + url.getHost()); 534 items.add(tr("Command")+": "+url.getPath()); 535 if (url.getQuery() != null) { 536 items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", "))); 537 } 538 return Utils.joinAsHtmlUnorderedList(items); 539 } 540 return null; 541 } 542}