001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005import static org.openstreetmap.josm.tools.I18n.trn;
006
007import java.io.IOException;
008import java.io.InputStream;
009import java.net.HttpURLConnection;
010import java.util.ArrayList;
011import java.util.Collection;
012import java.util.Collections;
013import java.util.HashSet;
014import java.util.Iterator;
015import java.util.LinkedHashMap;
016import java.util.LinkedHashSet;
017import java.util.List;
018import java.util.Map;
019import java.util.Map.Entry;
020import java.util.Set;
021import java.util.concurrent.Callable;
022import java.util.concurrent.CompletionService;
023import java.util.concurrent.ExecutionException;
024import java.util.concurrent.ExecutorCompletionService;
025import java.util.concurrent.ExecutorService;
026import java.util.concurrent.Executors;
027import java.util.concurrent.Future;
028import java.util.stream.Collectors;
029
030import org.openstreetmap.josm.data.Bounds;
031import org.openstreetmap.josm.data.osm.DataSet;
032import org.openstreetmap.josm.data.osm.DataSetMerger;
033import org.openstreetmap.josm.data.osm.Node;
034import org.openstreetmap.josm.data.osm.OsmPrimitive;
035import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
036import org.openstreetmap.josm.data.osm.PrimitiveId;
037import org.openstreetmap.josm.data.osm.Relation;
038import org.openstreetmap.josm.data.osm.RelationMember;
039import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
040import org.openstreetmap.josm.data.osm.Way;
041import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
042import org.openstreetmap.josm.gui.progress.ProgressMonitor;
043import org.openstreetmap.josm.spi.preferences.Config;
044import org.openstreetmap.josm.tools.Logging;
045import org.openstreetmap.josm.tools.Utils;
046
047/**
048 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called
049 * Multi Fetch API.
050 *
051 * Usage:
052 * <pre>
053 *    MultiFetchServerObjectReader reader = MultiFetchServerObjectReader()
054 *         .append(new Node(72343));
055 *    reader.parseOsm();
056 *    if (!reader.getMissingPrimitives().isEmpty()) {
057 *        Logging.info("There are missing primitives: " + reader.getMissingPrimitives());
058 *    }
059 *    if (!reader.getSkippedWays().isEmpty()) {
060 *       Logging.info("There are skipped ways: " + reader.getMissingPrimitives());
061 *    }
062 * </pre>
063 */
064public class MultiFetchServerObjectReader extends OsmServerReader {
065    /**
066     * the max. number of primitives retrieved in one step. Assuming IDs with 10 digits,
067     * this leads to a max. request URL of ~ 1900 Bytes ((10 digits +  1 Separator) * 170),
068     * which should be safe according to the
069     * <a href="https://web.archive.org/web/20190902193246/https://boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>.
070     */
071    private static final int MAX_IDS_PER_REQUEST = 170;
072
073    private final Set<Long> nodes;
074    private final Set<Long> ways;
075    private final Set<Long> relations;
076    private final Set<PrimitiveId> missingPrimitives;
077    private final DataSet outputDataSet;
078    protected final Map<OsmPrimitiveType, Set<Long>> primitivesMap;
079
080    protected boolean recurseDownRelations;
081    private boolean recurseDownAppended = true;
082
083    private ExecutorService exec;
084
085    /**
086     * Constructs a {@code MultiFetchServerObjectReader}.
087     */
088    protected MultiFetchServerObjectReader() {
089        nodes = new LinkedHashSet<>();
090        ways = new LinkedHashSet<>();
091        relations = new LinkedHashSet<>();
092        this.outputDataSet = new DataSet();
093        this.missingPrimitives = new LinkedHashSet<>();
094        primitivesMap = new LinkedHashMap<>();
095        primitivesMap.put(OsmPrimitiveType.RELATION, relations);
096        primitivesMap.put(OsmPrimitiveType.WAY, ways);
097        primitivesMap.put(OsmPrimitiveType.NODE, nodes);
098    }
099
100    /**
101     * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader}
102     * depending on the {@link OverpassDownloadReader#FOR_MULTI_FETCH preference}.
103     *
104     * @return a new instance
105     * @since 9241
106     */
107    public static MultiFetchServerObjectReader create() {
108        return create(OverpassDownloadReader.FOR_MULTI_FETCH.get());
109    }
110
111    /**
112     * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader}
113     * depending on the {@code fromMirror} parameter.
114     *
115     * @param fromMirror {@code false} for {@link MultiFetchServerObjectReader}, {@code true} for {@link MultiFetchOverpassObjectReader}
116     * @return a new instance
117     * @since 15520 (changed visibility)
118     */
119    public static MultiFetchServerObjectReader create(final boolean fromMirror) {
120        if (fromMirror) {
121            return new MultiFetchOverpassObjectReader();
122        } else {
123            return new MultiFetchServerObjectReader();
124        }
125    }
126
127    /**
128     * Remembers an {@link OsmPrimitive}'s id. The id will
129     * later be fetched as part of a Multi Get request.
130     *
131     * Ignore the id if it represents a new primitives.
132     *
133     * @param id  the id
134     */
135    public void append(PrimitiveId id) {
136        if (id.isNew()) return;
137        switch(id.getType()) {
138        case NODE: nodes.add(id.getUniqueId()); break;
139        case WAY: ways.add(id.getUniqueId()); break;
140        case RELATION: relations.add(id.getUniqueId()); break;
141        default: throw new AssertionError();
142        }
143    }
144
145    /**
146     * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server.
147     *
148     * @param ds the {@link DataSet} to which the primitive belongs
149     * @param id the primitive id
150     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
151     * {@link OsmPrimitiveType#RELATION RELATION}
152     * @return this
153     */
154    public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) {
155        OsmPrimitive p = ds.getPrimitiveById(id, type);
156        return append(p);
157    }
158
159    /**
160     * appends a {@link Node} id to the list of ids which will be fetched from the server.
161     *
162     * @param node  the node (ignored, if null)
163     * @return this
164     */
165    public MultiFetchServerObjectReader appendNode(Node node) {
166        if (node == null || node.isNew()) return this;
167        append(node.getPrimitiveId());
168        return this;
169    }
170
171    /**
172     * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server.
173     *
174     * @param way the way (ignored, if null)
175     * @return this
176     */
177    public MultiFetchServerObjectReader appendWay(Way way) {
178        if (way == null || way.isNew()) return this;
179        if (recurseDownAppended) {
180            append(way.getNodes());
181        }
182        append(way.getPrimitiveId());
183        return this;
184    }
185
186    /**
187     * appends a {@link Relation} id to the list of ids which will be fetched from the server.
188     *
189     * @param relation  the relation (ignored, if null)
190     * @return this
191     */
192    protected MultiFetchServerObjectReader appendRelation(Relation relation) {
193        if (relation == null || relation.isNew()) return this;
194        append(relation.getPrimitiveId());
195        if (recurseDownAppended) {
196            for (RelationMember member : relation.getMembers()) {
197                // avoid infinite recursion in case of cyclic dependencies in relations
198                if (OsmPrimitiveType.from(member.getMember()) == OsmPrimitiveType.RELATION
199                        && relations.contains(member.getMember().getId())) {
200                    continue;
201                }
202                if (!member.getMember().isIncomplete()) {
203                    append(member.getMember());
204                }
205            }
206        }
207        return this;
208    }
209
210    /**
211     * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server.
212     * @param primitive the primitive
213     * @return this
214     */
215    public MultiFetchServerObjectReader append(OsmPrimitive primitive) {
216        if (primitive instanceof Node) {
217            return appendNode((Node) primitive);
218        } else if (primitive instanceof Way) {
219            return appendWay((Way) primitive);
220        } else if (primitive instanceof Relation) {
221            return appendRelation((Relation) primitive);
222        }
223        return this;
224    }
225
226    /**
227     * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server.
228     *
229     * @param primitives  the list of primitives (ignored, if null)
230     * @return this
231     *
232     * @see #append(OsmPrimitive)
233     */
234    public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) {
235        if (primitives == null) return this;
236        primitives.forEach(this::append);
237        return this;
238    }
239
240    /**
241     * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and
242     * replies the subset. The extracted subset is removed from <code>ids</code>.
243     *
244     * @param ids a set of ids
245     * @return the subset of ids
246     */
247    protected Set<Long> extractIdPackage(Set<Long> ids) {
248        Set<Long> pkg = new HashSet<>();
249        if (ids.isEmpty())
250            return pkg;
251        if (ids.size() > MAX_IDS_PER_REQUEST) {
252            Iterator<Long> it = ids.iterator();
253            for (int i = 0; i < MAX_IDS_PER_REQUEST; i++) {
254                pkg.add(it.next());
255            }
256            ids.removeAll(pkg);
257        } else {
258            pkg.addAll(ids);
259            ids.clear();
260        }
261        return pkg;
262    }
263
264    /**
265     * builds the Multi Get request string for a set of ids and a given {@link OsmPrimitiveType}.
266     *
267     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
268     * {@link OsmPrimitiveType#RELATION RELATION}
269     * @param idPackage  the package of ids
270     * @return the request string
271     */
272    protected String buildRequestString(final OsmPrimitiveType type, Set<Long> idPackage) {
273        return type.getAPIName() + "s?" + type.getAPIName() + "s=" + idPackage.stream().map(String::valueOf).collect(Collectors.joining(","));
274    }
275
276    protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) {
277        for (Way w: from.getWays()) {
278            for (Node n: w.getNodes()) {
279                if (n.isIncomplete()) {
280                    nodes.add(n.getId());
281                }
282            }
283        }
284    }
285
286    /**
287     * merges the dataset <code>from</code> to {@link #outputDataSet}.
288     *
289     * @param from the other dataset
290     */
291    protected void merge(DataSet from) {
292        final DataSetMerger visitor = new DataSetMerger(outputDataSet, from);
293        visitor.merge();
294    }
295
296    /**
297     * fetches a set of ids of a given {@link OsmPrimitiveType} from the server
298     *
299     * @param ids the set of ids
300     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
301     * {@link OsmPrimitiveType#RELATION RELATION}
302     * @param progressMonitor progress monitor
303     * @throws OsmTransferException if an error occurs while communicating with the API server
304     */
305    protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException {
306        String msg;
307        final String baseUrl = getBaseUrl();
308        switch (type) {
309            // CHECKSTYLE.OFF: SingleSpaceSeparator
310            case NODE:     msg = tr("Fetching a package of nodes from ''{0}''",     baseUrl); break;
311            case WAY:      msg = tr("Fetching a package of ways from ''{0}''",      baseUrl); break;
312            case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break;
313            // CHECKSTYLE.ON: SingleSpaceSeparator
314            default: throw new AssertionError();
315        }
316        progressMonitor.setTicksCount(ids.size());
317        progressMonitor.setTicks(0);
318        // The complete set containing all primitives to fetch
319        Set<Long> toFetch = new HashSet<>(ids);
320        // Build a list of fetchers that will  download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each.
321        // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers.
322        int threadsNumber = Config.getPref().getInt("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS);
323        threadsNumber = Utils.clamp(threadsNumber, 1, OsmApi.MAX_DOWNLOAD_THREADS);
324        exec = Executors.newFixedThreadPool(
325                threadsNumber, Utils.newThreadFactory(getClass() + "-%d", Thread.NORM_PRIORITY));
326        CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec);
327        List<Future<FetchResult>> jobs = new ArrayList<>();
328        while (!toFetch.isEmpty() && !isCanceled()) {
329            jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor)));
330        }
331        // Run the fetchers
332        for (int i = 0; i < jobs.size() && !isCanceled(); i++) {
333            progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + '/' + progressMonitor.getTicksCount());
334            try {
335                FetchResult result = ecs.take().get();
336                if (result.rc404 != null) {
337                    List<Long> toSplit = new ArrayList<>(result.rc404);
338                    int n = toSplit.size() / 2;
339                    jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(0, n)), progressMonitor)));
340                    jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(n, toSplit.size())), progressMonitor)));
341                }
342                if (result.missingPrimitives != null) {
343                    missingPrimitives.addAll(result.missingPrimitives);
344                }
345                if (result.dataSet != null && !isCanceled()) {
346                    rememberNodesOfIncompleteWaysToLoad(result.dataSet);
347                    merge(result.dataSet);
348                }
349            } catch (InterruptedException | ExecutionException e) {
350                Logging.error(e);
351                if (e.getCause() instanceof OsmTransferException)
352                    throw (OsmTransferException) e.getCause();
353            }
354        }
355        exec.shutdown();
356        // Cancel requests if the user chose to
357        if (isCanceled()) {
358            for (Future<FetchResult> job : jobs) {
359                job.cancel(true);
360            }
361        }
362        exec = null;
363    }
364
365    /**
366     * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies
367     * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too!
368     * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies
369     * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if
370     * visible==false).
371     *
372     * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been
373     * found on  the server (the server response code was 404)
374     *
375     * @param progressMonitor progress monitor
376     * @return the parsed data
377     * @throws OsmTransferException if an error occurs while communicating with the API server
378     * @see #getMissingPrimitives()
379     *
380     */
381    @Override
382    public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
383        missingPrimitives.clear();
384        int n = nodes.size() + ways.size() + relations.size();
385        progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''",
386                "Downloading {0} objects from ''{1}''", n, n, getBaseUrl()));
387        try {
388            if (this instanceof MultiFetchOverpassObjectReader) {
389                // calculate a single request for all the objects
390                String request = MultiFetchOverpassObjectReader.genOverpassQuery(primitivesMap, true, false, recurseDownRelations);
391                if (isCanceled())
392                    return null;
393                OverpassDownloadReader reader = new OverpassDownloadReader(new Bounds(0, 0, 0, 0), getBaseUrl(), request);
394                DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false));
395                new DataSetMerger(outputDataSet, ds).merge();
396                checkMissing(outputDataSet, progressMonitor);
397            } else {
398                downloadRelations(progressMonitor);
399                if (isCanceled())
400                    return null;
401                fetchPrimitives(ways, OsmPrimitiveType.WAY, progressMonitor);
402                if (isCanceled())
403                    return null;
404                fetchPrimitives(nodes, OsmPrimitiveType.NODE, progressMonitor);
405            }
406            outputDataSet.deleteInvisible();
407            return outputDataSet;
408        } finally {
409            progressMonitor.finishTask();
410        }
411    }
412
413    /**
414     * Workaround for difference in Overpass API.
415     * As of now (version 7.55) Overpass api doesn't return invisible objects.
416     * Check if we have objects which do not appear in the dataset and fetch them from OSM instead.
417     * @param ds the dataset
418     * @param progressMonitor progress monitor
419     * @throws OsmTransferException if an error occurs while communicating with the API server
420     */
421    private void checkMissing(DataSet ds, ProgressMonitor progressMonitor) throws OsmTransferException {
422        Set<OsmPrimitive> missing = new LinkedHashSet<>();
423        for (Entry<OsmPrimitiveType, Set<Long>> e : primitivesMap.entrySet()) {
424            for (long id : e.getValue()) {
425                if (ds.getPrimitiveById(id, e.getKey()) == null)
426                    missing.add(e.getKey().newInstance(id, true));
427            }
428        }
429        if (isCanceled() || missing.isEmpty())
430            return;
431
432        MultiFetchServerObjectReader missingReader = MultiFetchServerObjectReader.create(false);
433        missingReader.setRecurseDownAppended(false);
434        missingReader.setRecurseDownRelations(false);
435        missingReader.append(missing);
436        DataSet mds = missingReader.parseOsm(progressMonitor.createSubTaskMonitor(missing.size(), false));
437        new DataSetMerger(ds, mds).merge();
438        missingPrimitives.addAll(missingReader.getMissingPrimitives());
439    }
440
441    /**
442     * Finds best way to download a set of relations.
443     * @param progressMonitor progress monitor
444     * @throws OsmTransferException if an error occurs while communicating with the API server
445     * @see #getMissingPrimitives()
446     */
447    private void downloadRelations(ProgressMonitor progressMonitor) throws OsmTransferException {
448        boolean removeIncomplete = outputDataSet.isEmpty();
449        Set<Long> toDownload = new LinkedHashSet<>(relations);
450        fetchPrimitives(toDownload, OsmPrimitiveType.RELATION, progressMonitor);
451        if (!recurseDownRelations) {
452            return;
453        }
454        // OSM multi-fetch api may return invisible objects, we don't try to get details for them
455        for (Relation r : outputDataSet.getRelations()) {
456            if (!r.isVisible()) {
457                toDownload.remove(r.getUniqueId());
458            } else if (removeIncomplete) {
459                outputDataSet.removePrimitive(r);
460            }
461        }
462
463        // fetch full info for all visible relations
464        for (long id : toDownload) {
465            if (isCanceled())
466                return;
467            OsmServerObjectReader reader = new OsmServerObjectReader(id, OsmPrimitiveType.RELATION, true/* full*/);
468            DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false));
469            merge(ds);
470        }
471    }
472
473    /**
474     * replies the set of ids of all primitives for which a fetch request to the
475     * server was submitted but which are not available from the server (the server
476     * replied a return code of 404)
477     *
478     * @return the set of ids of missing primitives
479     */
480    public Set<PrimitiveId> getMissingPrimitives() {
481        return missingPrimitives;
482    }
483
484    /**
485     * Should downloaded relations be complete?
486     * @param recurseDownRelations true: yes, recurse down to retrieve the members of the relation
487     * This will download sub relations, complete way members and nodes. Members of sub relations are not
488     * retrieved unless they are also members of the relations. See #18835.
489     * @return this
490     * @since 15811
491     */
492    public MultiFetchServerObjectReader setRecurseDownRelations(boolean recurseDownRelations) {
493        this.recurseDownRelations = recurseDownRelations;
494        return this;
495    }
496
497    /**
498     * Determine how appended objects are treated. By default, all children of an appended object are also appended.
499     * @param recurseAppended false: do not append known children of appended objects, i.e. all nodes of way and all members of a relation
500     * @return this
501     * @since 15811
502     */
503    public MultiFetchServerObjectReader setRecurseDownAppended(boolean recurseAppended) {
504        this.recurseDownAppended = recurseAppended;
505        return this;
506    }
507
508    /**
509     * The class holding the results given by {@link Fetcher}.
510     * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded.
511     */
512    protected static class FetchResult {
513
514        /**
515         * The resulting data set
516         */
517        public final DataSet dataSet;
518
519        /**
520         * The collection of primitive ids that could not have been loaded
521         */
522        public final Set<PrimitiveId> missingPrimitives;
523
524        private Set<Long> rc404;
525
526        /**
527         * Constructs a {@code FetchResult}
528         * @param dataSet The resulting data set
529         * @param missingPrimitives The collection of primitive ids that could not have been loaded
530         */
531        public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) {
532            this.dataSet = dataSet;
533            this.missingPrimitives = missingPrimitives;
534        }
535    }
536
537    /**
538     * The class that actually download data from OSM API.
539     * Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch).
540     * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance.
541     * @see FetchResult
542     */
543    protected class Fetcher extends OsmServerReader implements Callable<FetchResult> {
544
545        private final Set<Long> pkg;
546        private final OsmPrimitiveType type;
547        private final ProgressMonitor progressMonitor;
548
549        /**
550         * Constructs a {@code Fetcher}
551         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
552         * {@link OsmPrimitiveType#RELATION RELATION}
553         * @param idsPackage The set of primitives ids to fetch
554         * @param progressMonitor The progress monitor
555         */
556        public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) {
557            this.pkg = idsPackage;
558            this.type = type;
559            this.progressMonitor = progressMonitor;
560        }
561
562        @Override
563        public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
564            // This method is implemented because of the OsmServerReader inheritance, but not used,
565            // as the main target of this class is the call() method.
566            return fetch(progressMonitor).dataSet;
567        }
568
569        @Override
570        public FetchResult call() throws Exception {
571            return fetch(progressMonitor);
572        }
573
574        /**
575         * fetches the requested primitives and updates the specified progress monitor.
576         * @param progressMonitor the progress monitor
577         * @return the {@link FetchResult} of this operation
578         * @throws OsmTransferException if an error occurs while communicating with the API server
579         */
580        protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException {
581            try {
582                return multiGetIdPackage(type, pkg, progressMonitor);
583            } catch (OsmApiException e) {
584                if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
585                    if (pkg.size() > 4) {
586                        FetchResult res = new FetchResult(null, null);
587                        res.rc404 = pkg;
588                        return res;
589                    }
590                    if (pkg.size() == 1) {
591                        FetchResult res = new FetchResult(new DataSet(), new HashSet<PrimitiveId>());
592                        res.missingPrimitives.add(new SimplePrimitiveId(pkg.iterator().next(), type));
593                        return res;
594                    } else {
595                        Logging.info(tr("Server replied with response code 404, retrying with an individual request for each object."));
596                        return singleGetIdPackage(type, pkg, progressMonitor);
597                    }
598                } else {
599                    throw e;
600                }
601            }
602        }
603
604        @Override
605        protected String getBaseUrl() {
606            return MultiFetchServerObjectReader.this.getBaseUrl();
607        }
608
609        /**
610         * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}.
611         * The retrieved primitives are merged to {@link #outputDataSet}.
612         *
613         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
614         * {@link OsmPrimitiveType#RELATION RELATION}
615         * @param pkg the package of ids
616         * @param progressMonitor progress monitor
617         * @return the {@link FetchResult} of this operation
618         * @throws OsmTransferException if an error occurs while communicating with the API server
619         */
620        protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor)
621                throws OsmTransferException {
622            String request = buildRequestString(type, pkg);
623            FetchResult result = null;
624            try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
625                if (in == null) return null;
626                progressMonitor.subTask(tr("Downloading OSM data..."));
627                try {
628                    result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null);
629                } catch (IllegalDataException e) {
630                    throw new OsmTransferException(e);
631                }
632            } catch (IOException ex) {
633                Logging.warn(ex);
634                throw new OsmTransferException(ex);
635            }
636            return result;
637        }
638
639        /**
640         * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
641         * The retrieved primitive is merged to {@link #outputDataSet}.
642         *
643         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
644         * {@link OsmPrimitiveType#RELATION RELATION}
645         * @param id the id
646         * @param progressMonitor progress monitor
647         * @return the {@link DataSet} resulting of this operation
648         * @throws OsmTransferException if an error occurs while communicating with the API server
649         */
650        protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
651            String request = buildRequestString(type, Collections.singleton(id));
652            DataSet result = null;
653            try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
654                if (in == null) return null;
655                progressMonitor.subTask(tr("Downloading OSM data..."));
656                try {
657                    result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
658                } catch (IllegalDataException e) {
659                    throw new OsmTransferException(e);
660                }
661            } catch (IOException ex) {
662                Logging.warn(ex);
663            }
664            return result;
665        }
666
667        /**
668         * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
669         * The retrieved primitives are merged to {@link #outputDataSet}.
670         *
671         * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
672         * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
673         * Unfortunately, the server does not provide an error header or an error body for a 404 reply.
674         *
675         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
676         * {@link OsmPrimitiveType#RELATION RELATION}
677         * @param pkg the set of ids
678         * @param progressMonitor progress monitor
679         * @return the {@link FetchResult} of this operation
680         * @throws OsmTransferException if an error occurs while communicating with the API server
681         */
682        protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor)
683                throws OsmTransferException {
684            FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>());
685            String baseUrl = OsmApi.getOsmApi().getBaseUrl();
686            for (long id : pkg) {
687                try {
688                    String msg;
689                    switch (type) {
690                        // CHECKSTYLE.OFF: SingleSpaceSeparator
691                        case NODE:     msg = tr("Fetching node with id {0} from ''{1}''",     id, baseUrl); break;
692                        case WAY:      msg = tr("Fetching way with id {0} from ''{1}''",      id, baseUrl); break;
693                        case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break;
694                        // CHECKSTYLE.ON: SingleSpaceSeparator
695                        default: throw new AssertionError();
696                    }
697                    progressMonitor.setCustomText(msg);
698                    result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor));
699                } catch (OsmApiException e) {
700                    if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
701                        Logging.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
702                        result.missingPrimitives.add(new SimplePrimitiveId(id, type));
703                    } else {
704                        throw e;
705                    }
706                }
707            }
708            return result;
709        }
710    }
711
712    @Override
713    public void cancel() {
714        super.cancel();
715        if (exec != null)
716            exec.shutdownNow();
717    }
718}