001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005import static org.openstreetmap.josm.tools.I18n.trn; 006 007import java.io.IOException; 008import java.io.InputStream; 009import java.net.HttpURLConnection; 010import java.util.ArrayList; 011import java.util.Collection; 012import java.util.Collections; 013import java.util.HashSet; 014import java.util.Iterator; 015import java.util.LinkedHashMap; 016import java.util.LinkedHashSet; 017import java.util.List; 018import java.util.Map; 019import java.util.Map.Entry; 020import java.util.Set; 021import java.util.concurrent.Callable; 022import java.util.concurrent.CompletionService; 023import java.util.concurrent.ExecutionException; 024import java.util.concurrent.ExecutorCompletionService; 025import java.util.concurrent.ExecutorService; 026import java.util.concurrent.Executors; 027import java.util.concurrent.Future; 028import java.util.stream.Collectors; 029 030import org.openstreetmap.josm.data.Bounds; 031import org.openstreetmap.josm.data.osm.DataSet; 032import org.openstreetmap.josm.data.osm.DataSetMerger; 033import org.openstreetmap.josm.data.osm.Node; 034import org.openstreetmap.josm.data.osm.OsmPrimitive; 035import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 036import org.openstreetmap.josm.data.osm.PrimitiveId; 037import org.openstreetmap.josm.data.osm.Relation; 038import org.openstreetmap.josm.data.osm.RelationMember; 039import org.openstreetmap.josm.data.osm.SimplePrimitiveId; 040import org.openstreetmap.josm.data.osm.Way; 041import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 042import org.openstreetmap.josm.gui.progress.ProgressMonitor; 043import org.openstreetmap.josm.spi.preferences.Config; 044import org.openstreetmap.josm.tools.Logging; 045import org.openstreetmap.josm.tools.Utils; 046 047/** 048 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called 049 * Multi Fetch API. 050 * 051 * Usage: 052 * <pre> 053 * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader() 054 * .append(new Node(72343)); 055 * reader.parseOsm(); 056 * if (!reader.getMissingPrimitives().isEmpty()) { 057 * Logging.info("There are missing primitives: " + reader.getMissingPrimitives()); 058 * } 059 * if (!reader.getSkippedWays().isEmpty()) { 060 * Logging.info("There are skipped ways: " + reader.getMissingPrimitives()); 061 * } 062 * </pre> 063 */ 064public class MultiFetchServerObjectReader extends OsmServerReader { 065 /** 066 * the max. number of primitives retrieved in one step. Assuming IDs with 10 digits, 067 * this leads to a max. request URL of ~ 1900 Bytes ((10 digits + 1 Separator) * 170), 068 * which should be safe according to the 069 * <a href="https://web.archive.org/web/20190902193246/https://boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>. 070 */ 071 private static final int MAX_IDS_PER_REQUEST = 170; 072 073 private final Set<Long> nodes; 074 private final Set<Long> ways; 075 private final Set<Long> relations; 076 private final Set<PrimitiveId> missingPrimitives; 077 private final DataSet outputDataSet; 078 protected final Map<OsmPrimitiveType, Set<Long>> primitivesMap; 079 080 protected boolean recurseDownRelations; 081 private boolean recurseDownAppended = true; 082 083 private ExecutorService exec; 084 085 /** 086 * Constructs a {@code MultiFetchServerObjectReader}. 087 */ 088 protected MultiFetchServerObjectReader() { 089 nodes = new LinkedHashSet<>(); 090 ways = new LinkedHashSet<>(); 091 relations = new LinkedHashSet<>(); 092 this.outputDataSet = new DataSet(); 093 this.missingPrimitives = new LinkedHashSet<>(); 094 primitivesMap = new LinkedHashMap<>(); 095 primitivesMap.put(OsmPrimitiveType.RELATION, relations); 096 primitivesMap.put(OsmPrimitiveType.WAY, ways); 097 primitivesMap.put(OsmPrimitiveType.NODE, nodes); 098 } 099 100 /** 101 * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader} 102 * depending on the {@link OverpassDownloadReader#FOR_MULTI_FETCH preference}. 103 * 104 * @return a new instance 105 * @since 9241 106 */ 107 public static MultiFetchServerObjectReader create() { 108 return create(OverpassDownloadReader.FOR_MULTI_FETCH.get()); 109 } 110 111 /** 112 * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader} 113 * depending on the {@code fromMirror} parameter. 114 * 115 * @param fromMirror {@code false} for {@link MultiFetchServerObjectReader}, {@code true} for {@link MultiFetchOverpassObjectReader} 116 * @return a new instance 117 * @since 15520 (changed visibility) 118 */ 119 public static MultiFetchServerObjectReader create(final boolean fromMirror) { 120 if (fromMirror) { 121 return new MultiFetchOverpassObjectReader(); 122 } else { 123 return new MultiFetchServerObjectReader(); 124 } 125 } 126 127 /** 128 * Remembers an {@link OsmPrimitive}'s id. The id will 129 * later be fetched as part of a Multi Get request. 130 * 131 * Ignore the id if it represents a new primitives. 132 * 133 * @param id the id 134 */ 135 public void append(PrimitiveId id) { 136 if (id.isNew()) return; 137 switch(id.getType()) { 138 case NODE: nodes.add(id.getUniqueId()); break; 139 case WAY: ways.add(id.getUniqueId()); break; 140 case RELATION: relations.add(id.getUniqueId()); break; 141 default: throw new AssertionError(); 142 } 143 } 144 145 /** 146 * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server. 147 * 148 * @param ds the {@link DataSet} to which the primitive belongs 149 * @param id the primitive id 150 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 151 * {@link OsmPrimitiveType#RELATION RELATION} 152 * @return this 153 */ 154 public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) { 155 OsmPrimitive p = ds.getPrimitiveById(id, type); 156 return append(p); 157 } 158 159 /** 160 * appends a {@link Node} id to the list of ids which will be fetched from the server. 161 * 162 * @param node the node (ignored, if null) 163 * @return this 164 */ 165 public MultiFetchServerObjectReader appendNode(Node node) { 166 if (node == null || node.isNew()) return this; 167 append(node.getPrimitiveId()); 168 return this; 169 } 170 171 /** 172 * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server. 173 * 174 * @param way the way (ignored, if null) 175 * @return this 176 */ 177 public MultiFetchServerObjectReader appendWay(Way way) { 178 if (way == null || way.isNew()) return this; 179 if (recurseDownAppended) { 180 append(way.getNodes()); 181 } 182 append(way.getPrimitiveId()); 183 return this; 184 } 185 186 /** 187 * appends a {@link Relation} id to the list of ids which will be fetched from the server. 188 * 189 * @param relation the relation (ignored, if null) 190 * @return this 191 */ 192 protected MultiFetchServerObjectReader appendRelation(Relation relation) { 193 if (relation == null || relation.isNew()) return this; 194 append(relation.getPrimitiveId()); 195 if (recurseDownAppended) { 196 for (RelationMember member : relation.getMembers()) { 197 // avoid infinite recursion in case of cyclic dependencies in relations 198 if (OsmPrimitiveType.from(member.getMember()) == OsmPrimitiveType.RELATION 199 && relations.contains(member.getMember().getId())) { 200 continue; 201 } 202 if (!member.getMember().isIncomplete()) { 203 append(member.getMember()); 204 } 205 } 206 } 207 return this; 208 } 209 210 /** 211 * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server. 212 * @param primitive the primitive 213 * @return this 214 */ 215 public MultiFetchServerObjectReader append(OsmPrimitive primitive) { 216 if (primitive instanceof Node) { 217 return appendNode((Node) primitive); 218 } else if (primitive instanceof Way) { 219 return appendWay((Way) primitive); 220 } else if (primitive instanceof Relation) { 221 return appendRelation((Relation) primitive); 222 } 223 return this; 224 } 225 226 /** 227 * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server. 228 * 229 * @param primitives the list of primitives (ignored, if null) 230 * @return this 231 * 232 * @see #append(OsmPrimitive) 233 */ 234 public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) { 235 if (primitives == null) return this; 236 primitives.forEach(this::append); 237 return this; 238 } 239 240 /** 241 * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and 242 * replies the subset. The extracted subset is removed from <code>ids</code>. 243 * 244 * @param ids a set of ids 245 * @return the subset of ids 246 */ 247 protected Set<Long> extractIdPackage(Set<Long> ids) { 248 Set<Long> pkg = new HashSet<>(); 249 if (ids.isEmpty()) 250 return pkg; 251 if (ids.size() > MAX_IDS_PER_REQUEST) { 252 Iterator<Long> it = ids.iterator(); 253 for (int i = 0; i < MAX_IDS_PER_REQUEST; i++) { 254 pkg.add(it.next()); 255 } 256 ids.removeAll(pkg); 257 } else { 258 pkg.addAll(ids); 259 ids.clear(); 260 } 261 return pkg; 262 } 263 264 /** 265 * builds the Multi Get request string for a set of ids and a given {@link OsmPrimitiveType}. 266 * 267 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 268 * {@link OsmPrimitiveType#RELATION RELATION} 269 * @param idPackage the package of ids 270 * @return the request string 271 */ 272 protected String buildRequestString(final OsmPrimitiveType type, Set<Long> idPackage) { 273 return type.getAPIName() + "s?" + type.getAPIName() + "s=" + idPackage.stream().map(String::valueOf).collect(Collectors.joining(",")); 274 } 275 276 protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) { 277 for (Way w: from.getWays()) { 278 for (Node n: w.getNodes()) { 279 if (n.isIncomplete()) { 280 nodes.add(n.getId()); 281 } 282 } 283 } 284 } 285 286 /** 287 * merges the dataset <code>from</code> to {@link #outputDataSet}. 288 * 289 * @param from the other dataset 290 */ 291 protected void merge(DataSet from) { 292 final DataSetMerger visitor = new DataSetMerger(outputDataSet, from); 293 visitor.merge(); 294 } 295 296 /** 297 * fetches a set of ids of a given {@link OsmPrimitiveType} from the server 298 * 299 * @param ids the set of ids 300 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 301 * {@link OsmPrimitiveType#RELATION RELATION} 302 * @param progressMonitor progress monitor 303 * @throws OsmTransferException if an error occurs while communicating with the API server 304 */ 305 protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException { 306 String msg; 307 final String baseUrl = getBaseUrl(); 308 switch (type) { 309 // CHECKSTYLE.OFF: SingleSpaceSeparator 310 case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break; 311 case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break; 312 case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break; 313 // CHECKSTYLE.ON: SingleSpaceSeparator 314 default: throw new AssertionError(); 315 } 316 progressMonitor.setTicksCount(ids.size()); 317 progressMonitor.setTicks(0); 318 // The complete set containing all primitives to fetch 319 Set<Long> toFetch = new HashSet<>(ids); 320 // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each. 321 // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers. 322 int threadsNumber = Config.getPref().getInt("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS); 323 threadsNumber = Utils.clamp(threadsNumber, 1, OsmApi.MAX_DOWNLOAD_THREADS); 324 exec = Executors.newFixedThreadPool( 325 threadsNumber, Utils.newThreadFactory(getClass() + "-%d", Thread.NORM_PRIORITY)); 326 CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec); 327 List<Future<FetchResult>> jobs = new ArrayList<>(); 328 while (!toFetch.isEmpty() && !isCanceled()) { 329 jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor))); 330 } 331 // Run the fetchers 332 for (int i = 0; i < jobs.size() && !isCanceled(); i++) { 333 progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + '/' + progressMonitor.getTicksCount()); 334 try { 335 FetchResult result = ecs.take().get(); 336 if (result.rc404 != null) { 337 List<Long> toSplit = new ArrayList<>(result.rc404); 338 int n = toSplit.size() / 2; 339 jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(0, n)), progressMonitor))); 340 jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(n, toSplit.size())), progressMonitor))); 341 } 342 if (result.missingPrimitives != null) { 343 missingPrimitives.addAll(result.missingPrimitives); 344 } 345 if (result.dataSet != null && !isCanceled()) { 346 rememberNodesOfIncompleteWaysToLoad(result.dataSet); 347 merge(result.dataSet); 348 } 349 } catch (InterruptedException | ExecutionException e) { 350 Logging.error(e); 351 if (e.getCause() instanceof OsmTransferException) 352 throw (OsmTransferException) e.getCause(); 353 } 354 } 355 exec.shutdown(); 356 // Cancel requests if the user chose to 357 if (isCanceled()) { 358 for (Future<FetchResult> job : jobs) { 359 job.cancel(true); 360 } 361 } 362 exec = null; 363 } 364 365 /** 366 * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies 367 * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too! 368 * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies 369 * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if 370 * visible==false). 371 * 372 * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been 373 * found on the server (the server response code was 404) 374 * 375 * @param progressMonitor progress monitor 376 * @return the parsed data 377 * @throws OsmTransferException if an error occurs while communicating with the API server 378 * @see #getMissingPrimitives() 379 * 380 */ 381 @Override 382 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 383 missingPrimitives.clear(); 384 int n = nodes.size() + ways.size() + relations.size(); 385 progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''", 386 "Downloading {0} objects from ''{1}''", n, n, getBaseUrl())); 387 try { 388 if (this instanceof MultiFetchOverpassObjectReader) { 389 // calculate a single request for all the objects 390 String request = MultiFetchOverpassObjectReader.genOverpassQuery(primitivesMap, true, false, recurseDownRelations); 391 if (isCanceled()) 392 return null; 393 OverpassDownloadReader reader = new OverpassDownloadReader(new Bounds(0, 0, 0, 0), getBaseUrl(), request); 394 DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); 395 new DataSetMerger(outputDataSet, ds).merge(); 396 checkMissing(outputDataSet, progressMonitor); 397 } else { 398 downloadRelations(progressMonitor); 399 if (isCanceled()) 400 return null; 401 fetchPrimitives(ways, OsmPrimitiveType.WAY, progressMonitor); 402 if (isCanceled()) 403 return null; 404 fetchPrimitives(nodes, OsmPrimitiveType.NODE, progressMonitor); 405 } 406 outputDataSet.deleteInvisible(); 407 return outputDataSet; 408 } finally { 409 progressMonitor.finishTask(); 410 } 411 } 412 413 /** 414 * Workaround for difference in Overpass API. 415 * As of now (version 7.55) Overpass api doesn't return invisible objects. 416 * Check if we have objects which do not appear in the dataset and fetch them from OSM instead. 417 * @param ds the dataset 418 * @param progressMonitor progress monitor 419 * @throws OsmTransferException if an error occurs while communicating with the API server 420 */ 421 private void checkMissing(DataSet ds, ProgressMonitor progressMonitor) throws OsmTransferException { 422 Set<OsmPrimitive> missing = new LinkedHashSet<>(); 423 for (Entry<OsmPrimitiveType, Set<Long>> e : primitivesMap.entrySet()) { 424 for (long id : e.getValue()) { 425 if (ds.getPrimitiveById(id, e.getKey()) == null) 426 missing.add(e.getKey().newInstance(id, true)); 427 } 428 } 429 if (isCanceled() || missing.isEmpty()) 430 return; 431 432 MultiFetchServerObjectReader missingReader = MultiFetchServerObjectReader.create(false); 433 missingReader.setRecurseDownAppended(false); 434 missingReader.setRecurseDownRelations(false); 435 missingReader.append(missing); 436 DataSet mds = missingReader.parseOsm(progressMonitor.createSubTaskMonitor(missing.size(), false)); 437 new DataSetMerger(ds, mds).merge(); 438 missingPrimitives.addAll(missingReader.getMissingPrimitives()); 439 } 440 441 /** 442 * Finds best way to download a set of relations. 443 * @param progressMonitor progress monitor 444 * @throws OsmTransferException if an error occurs while communicating with the API server 445 * @see #getMissingPrimitives() 446 */ 447 private void downloadRelations(ProgressMonitor progressMonitor) throws OsmTransferException { 448 boolean removeIncomplete = outputDataSet.isEmpty(); 449 Set<Long> toDownload = new LinkedHashSet<>(relations); 450 fetchPrimitives(toDownload, OsmPrimitiveType.RELATION, progressMonitor); 451 if (!recurseDownRelations) { 452 return; 453 } 454 // OSM multi-fetch api may return invisible objects, we don't try to get details for them 455 for (Relation r : outputDataSet.getRelations()) { 456 if (!r.isVisible()) { 457 toDownload.remove(r.getUniqueId()); 458 } else if (removeIncomplete) { 459 outputDataSet.removePrimitive(r); 460 } 461 } 462 463 // fetch full info for all visible relations 464 for (long id : toDownload) { 465 if (isCanceled()) 466 return; 467 OsmServerObjectReader reader = new OsmServerObjectReader(id, OsmPrimitiveType.RELATION, true/* full*/); 468 DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); 469 merge(ds); 470 } 471 } 472 473 /** 474 * replies the set of ids of all primitives for which a fetch request to the 475 * server was submitted but which are not available from the server (the server 476 * replied a return code of 404) 477 * 478 * @return the set of ids of missing primitives 479 */ 480 public Set<PrimitiveId> getMissingPrimitives() { 481 return missingPrimitives; 482 } 483 484 /** 485 * Should downloaded relations be complete? 486 * @param recurseDownRelations true: yes, recurse down to retrieve the members of the relation 487 * This will download sub relations, complete way members and nodes. Members of sub relations are not 488 * retrieved unless they are also members of the relations. See #18835. 489 * @return this 490 * @since 15811 491 */ 492 public MultiFetchServerObjectReader setRecurseDownRelations(boolean recurseDownRelations) { 493 this.recurseDownRelations = recurseDownRelations; 494 return this; 495 } 496 497 /** 498 * Determine how appended objects are treated. By default, all children of an appended object are also appended. 499 * @param recurseAppended false: do not append known children of appended objects, i.e. all nodes of way and all members of a relation 500 * @return this 501 * @since 15811 502 */ 503 public MultiFetchServerObjectReader setRecurseDownAppended(boolean recurseAppended) { 504 this.recurseDownAppended = recurseAppended; 505 return this; 506 } 507 508 /** 509 * The class holding the results given by {@link Fetcher}. 510 * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded. 511 */ 512 protected static class FetchResult { 513 514 /** 515 * The resulting data set 516 */ 517 public final DataSet dataSet; 518 519 /** 520 * The collection of primitive ids that could not have been loaded 521 */ 522 public final Set<PrimitiveId> missingPrimitives; 523 524 private Set<Long> rc404; 525 526 /** 527 * Constructs a {@code FetchResult} 528 * @param dataSet The resulting data set 529 * @param missingPrimitives The collection of primitive ids that could not have been loaded 530 */ 531 public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) { 532 this.dataSet = dataSet; 533 this.missingPrimitives = missingPrimitives; 534 } 535 } 536 537 /** 538 * The class that actually download data from OSM API. 539 * Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch). 540 * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance. 541 * @see FetchResult 542 */ 543 protected class Fetcher extends OsmServerReader implements Callable<FetchResult> { 544 545 private final Set<Long> pkg; 546 private final OsmPrimitiveType type; 547 private final ProgressMonitor progressMonitor; 548 549 /** 550 * Constructs a {@code Fetcher} 551 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 552 * {@link OsmPrimitiveType#RELATION RELATION} 553 * @param idsPackage The set of primitives ids to fetch 554 * @param progressMonitor The progress monitor 555 */ 556 public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) { 557 this.pkg = idsPackage; 558 this.type = type; 559 this.progressMonitor = progressMonitor; 560 } 561 562 @Override 563 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 564 // This method is implemented because of the OsmServerReader inheritance, but not used, 565 // as the main target of this class is the call() method. 566 return fetch(progressMonitor).dataSet; 567 } 568 569 @Override 570 public FetchResult call() throws Exception { 571 return fetch(progressMonitor); 572 } 573 574 /** 575 * fetches the requested primitives and updates the specified progress monitor. 576 * @param progressMonitor the progress monitor 577 * @return the {@link FetchResult} of this operation 578 * @throws OsmTransferException if an error occurs while communicating with the API server 579 */ 580 protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException { 581 try { 582 return multiGetIdPackage(type, pkg, progressMonitor); 583 } catch (OsmApiException e) { 584 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 585 if (pkg.size() > 4) { 586 FetchResult res = new FetchResult(null, null); 587 res.rc404 = pkg; 588 return res; 589 } 590 if (pkg.size() == 1) { 591 FetchResult res = new FetchResult(new DataSet(), new HashSet<PrimitiveId>()); 592 res.missingPrimitives.add(new SimplePrimitiveId(pkg.iterator().next(), type)); 593 return res; 594 } else { 595 Logging.info(tr("Server replied with response code 404, retrying with an individual request for each object.")); 596 return singleGetIdPackage(type, pkg, progressMonitor); 597 } 598 } else { 599 throw e; 600 } 601 } 602 } 603 604 @Override 605 protected String getBaseUrl() { 606 return MultiFetchServerObjectReader.this.getBaseUrl(); 607 } 608 609 /** 610 * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}. 611 * The retrieved primitives are merged to {@link #outputDataSet}. 612 * 613 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 614 * {@link OsmPrimitiveType#RELATION RELATION} 615 * @param pkg the package of ids 616 * @param progressMonitor progress monitor 617 * @return the {@link FetchResult} of this operation 618 * @throws OsmTransferException if an error occurs while communicating with the API server 619 */ 620 protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) 621 throws OsmTransferException { 622 String request = buildRequestString(type, pkg); 623 FetchResult result = null; 624 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 625 if (in == null) return null; 626 progressMonitor.subTask(tr("Downloading OSM data...")); 627 try { 628 result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null); 629 } catch (IllegalDataException e) { 630 throw new OsmTransferException(e); 631 } 632 } catch (IOException ex) { 633 Logging.warn(ex); 634 throw new OsmTransferException(ex); 635 } 636 return result; 637 } 638 639 /** 640 * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}. 641 * The retrieved primitive is merged to {@link #outputDataSet}. 642 * 643 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 644 * {@link OsmPrimitiveType#RELATION RELATION} 645 * @param id the id 646 * @param progressMonitor progress monitor 647 * @return the {@link DataSet} resulting of this operation 648 * @throws OsmTransferException if an error occurs while communicating with the API server 649 */ 650 protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException { 651 String request = buildRequestString(type, Collections.singleton(id)); 652 DataSet result = null; 653 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 654 if (in == null) return null; 655 progressMonitor.subTask(tr("Downloading OSM data...")); 656 try { 657 result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false)); 658 } catch (IllegalDataException e) { 659 throw new OsmTransferException(e); 660 } 661 } catch (IOException ex) { 662 Logging.warn(ex); 663 } 664 return result; 665 } 666 667 /** 668 * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}. 669 * The retrieved primitives are merged to {@link #outputDataSet}. 670 * 671 * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404). 672 * If the set is fetched with this method it is possible to find out which of the ids doesn't exist. 673 * Unfortunately, the server does not provide an error header or an error body for a 404 reply. 674 * 675 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 676 * {@link OsmPrimitiveType#RELATION RELATION} 677 * @param pkg the set of ids 678 * @param progressMonitor progress monitor 679 * @return the {@link FetchResult} of this operation 680 * @throws OsmTransferException if an error occurs while communicating with the API server 681 */ 682 protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) 683 throws OsmTransferException { 684 FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>()); 685 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 686 for (long id : pkg) { 687 try { 688 String msg; 689 switch (type) { 690 // CHECKSTYLE.OFF: SingleSpaceSeparator 691 case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break; 692 case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break; 693 case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break; 694 // CHECKSTYLE.ON: SingleSpaceSeparator 695 default: throw new AssertionError(); 696 } 697 progressMonitor.setCustomText(msg); 698 result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor)); 699 } catch (OsmApiException e) { 700 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 701 Logging.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id))); 702 result.missingPrimitives.add(new SimplePrimitiveId(id, type)); 703 } else { 704 throw e; 705 } 706 } 707 } 708 return result; 709 } 710 } 711 712 @Override 713 public void cancel() { 714 super.cancel(); 715 if (exec != null) 716 exec.shutdownNow(); 717 } 718}