001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.io.InputStream;
008import java.nio.charset.StandardCharsets;
009import java.time.Duration;
010import java.time.LocalDateTime;
011import java.time.Period;
012import java.time.ZoneId;
013import java.time.ZoneOffset;
014import java.time.format.DateTimeParseException;
015import java.util.Arrays;
016import java.util.EnumMap;
017import java.util.List;
018import java.util.Locale;
019import java.util.Map;
020import java.util.NoSuchElementException;
021import java.util.Objects;
022import java.util.concurrent.ConcurrentHashMap;
023import java.util.concurrent.TimeUnit;
024import java.util.regex.Matcher;
025import java.util.regex.Pattern;
026
027import javax.xml.stream.XMLStreamConstants;
028import javax.xml.stream.XMLStreamException;
029
030import org.openstreetmap.josm.data.Bounds;
031import org.openstreetmap.josm.data.DataSource;
032import org.openstreetmap.josm.data.coor.LatLon;
033import org.openstreetmap.josm.data.osm.BBox;
034import org.openstreetmap.josm.data.osm.DataSet;
035import org.openstreetmap.josm.data.osm.DataSetMerger;
036import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
037import org.openstreetmap.josm.data.osm.PrimitiveId;
038import org.openstreetmap.josm.data.preferences.BooleanProperty;
039import org.openstreetmap.josm.data.preferences.ListProperty;
040import org.openstreetmap.josm.data.preferences.StringProperty;
041import org.openstreetmap.josm.gui.download.OverpassDownloadSource;
042import org.openstreetmap.josm.gui.progress.ProgressMonitor;
043import org.openstreetmap.josm.io.NameFinder.SearchResult;
044import org.openstreetmap.josm.tools.HttpClient;
045import org.openstreetmap.josm.tools.Logging;
046import org.openstreetmap.josm.tools.UncheckedParseException;
047import org.openstreetmap.josm.tools.Utils;
048import org.openstreetmap.josm.tools.date.DateUtils;
049
050/**
051 * Read content from an Overpass server.
052 *
053 * @since 8744
054 */
055public class OverpassDownloadReader extends BoundingBoxDownloader {
056
057    /**
058     * Property for current Overpass server.
059     * @since 12816
060     */
061    public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server",
062            "https://overpass-api.de/api/");
063    /**
064     * Property for list of known Overpass servers.
065     * @since 12816
066     */
067    public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers",
068            Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/"));
069    /**
070     * Property to determine if Overpass API should be used for multi-fetch download.
071     * @since 12816
072     */
073    public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false);
074
075    private static final String DATA_PREFIX = "?data=";
076
077    static final class OverpassOsmReader extends OsmReader {
078        @Override
079        protected void parseUnknown(boolean printWarning) throws XMLStreamException {
080            if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
081                final String text = parser.getElementText();
082                if (text.contains("runtime error")) {
083                    throw new XMLStreamException(text);
084                }
085            }
086            super.parseUnknown(printWarning);
087        }
088    }
089
090    static final class OverpassOsmJsonReader extends OsmJsonReader {
091
092    }
093
094    /**
095     * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
096     * @since 11916
097     */
098    public enum OverpassOutputFormat {
099        /** Default output format: plain OSM XML */
100        OSM_XML("xml"),
101        /** OSM JSON format (not GeoJson) */
102        OSM_JSON("json"),
103        /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
104        CSV("csv"),
105        /** Custom, see https://overpass-api.de/output_formats.html#custom */
106        CUSTOM("custom"),
107        /** Popup, see https://overpass-api.de/output_formats.html#popup */
108        POPUP("popup"),
109        /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
110        PBF("pbf");
111
112        private final String directive;
113
114        OverpassOutputFormat(String directive) {
115            this.directive = directive;
116        }
117
118        /**
119         * Returns the directive used in {@code [out:<directive>]} statement.
120         * @return the directive used in {@code [out:<directive>]} statement
121         */
122        public String getDirective() {
123            return directive;
124        }
125
126        /**
127         * Returns the {@code OverpassOutputFormat} matching the given directive.
128         * @param directive directive used in {@code [out:<directive>]} statement
129         * @return {@code OverpassOutputFormat} matching the given directive
130         * @throws IllegalArgumentException in case of invalid directive
131         */
132        static OverpassOutputFormat from(String directive) {
133            for (OverpassOutputFormat oof : values()) {
134                if (oof.directive.equals(directive)) {
135                    return oof;
136                }
137            }
138            throw new IllegalArgumentException(directive);
139        }
140    }
141
142    static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
143
144    static final Map<OverpassOutputFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
145
146    final String overpassServer;
147    final String overpassQuery;
148
149    /**
150     * Constructs a new {@code OverpassDownloadReader}.
151     *
152     * @param downloadArea   The area to download
153     * @param overpassServer The Overpass server to use
154     * @param overpassQuery  The Overpass query
155     */
156    public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
157        super(downloadArea);
158        setDoAuthenticate(false);
159        this.overpassServer = overpassServer;
160        this.overpassQuery = overpassQuery.trim();
161    }
162
163    /**
164     * Registers an OSM reader for the given Overpass output format.
165     * @param format Overpass output format
166     * @param readerClass OSM reader class
167     * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
168     */
169    public static final Class<? extends AbstractReader> registerOverpassOutputFormatReader(
170            OverpassOutputFormat format, Class<? extends AbstractReader> readerClass) {
171        return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
172    }
173
174    static {
175        registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_XML, OverpassOsmReader.class);
176        registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_JSON, OverpassOsmJsonReader.class);
177    }
178
179    @Override
180    protected String getBaseUrl() {
181        return overpassServer;
182    }
183
184    @Override
185    protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
186        if (overpassQuery.isEmpty())
187            return super.getRequestForBbox(lon1, lat1, lon2, lat2);
188        else {
189            final String query = this.overpassQuery
190                    .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
191                    .replace("{{center}}", center(lon1, lat1, lon2, lat2));
192            final String expandedOverpassQuery = expandExtendedQueries(query);
193            return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
194        }
195    }
196
197    /**
198     * Evaluates some features of overpass turbo extended query syntax.
199     * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
200     * @param query unexpanded query
201     * @return expanded query
202     */
203    static String expandExtendedQueries(String query) {
204        final StringBuffer sb = new StringBuffer();
205        final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
206        while (matcher.find()) {
207            try {
208                switch (matcher.group(1)) {
209                    case "date":
210                        matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now(ZoneId.systemDefault())));
211                        break;
212                    case "geocodeArea":
213                        matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
214                        break;
215                    case "geocodeBbox":
216                        matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
217                        break;
218                    case "geocodeCoords":
219                        matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
220                        break;
221                    case "geocodeId":
222                        matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
223                        break;
224                    default:
225                        Logging.warn("Unsupported syntax: " + matcher.group(1));
226                }
227            } catch (UncheckedParseException | DateTimeParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
228                final String msg = tr("Failed to evaluate {0}", matcher.group());
229                Logging.log(Logging.LEVEL_WARN, msg, ex);
230                matcher.appendReplacement(sb, "// " + msg + "\n");
231            }
232        }
233        matcher.appendTail(sb);
234        return sb.toString();
235    }
236
237    static String bbox(double lon1, double lat1, double lon2, double lat2) {
238        return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
239    }
240
241    static String center(double lon1, double lat1, double lon2, double lat2) {
242        LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
243        return c.lat()+ "," + c.lon();
244    }
245
246    static String date(String dateOrHumanDuration, LocalDateTime from) {
247        try {
248            return DateUtils.parseInstant(dateOrHumanDuration).toString();
249        } catch (UncheckedParseException e) {
250            Logging.trace(e);
251        }
252        return duration(dateOrHumanDuration, from);
253    }
254
255    static String duration(String humanDuration, LocalDateTime from) {
256        // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
257        String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
258                .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
259                .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
260        Matcher matcher = Pattern.compile(
261                "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
262                "((?:[0-9]+D)?)" +
263                "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
264        boolean javaPer = false;
265        boolean javaDur = false;
266        if (matcher.matches()) {
267            javaPer = !Utils.isEmpty(matcher.group(1));
268            javaDur = !Utils.isEmpty(matcher.group(3));
269            duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
270            if (javaDur) {
271                duration += 'T' + matcher.group(3);
272            }
273        }
274
275        // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
276        // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
277        Period p = null;
278        Duration d = null;
279        int idx = duration.indexOf('T');
280        if (javaPer) {
281            p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
282        }
283        if (javaDur) {
284            d = Duration.parse(javaPer ? 'P' + duration.substring(idx) : duration);
285        } else if (!javaPer) {
286            d = Duration.parse(duration);
287        }
288
289        // Now that period and duration are known, compute the correct date/time
290        LocalDateTime dt = from;
291        if (p != null) {
292            dt = dt.minus(p);
293        }
294        if (d != null) {
295            dt = dt.minus(d);
296        }
297
298        // Returns the date/time formatted in ISO 8601
299        return dt.toInstant(ZoneOffset.UTC).toString();
300    }
301
302    private static SearchResult searchName(String area) throws IOException {
303        return searchName(NameFinder.queryNominatim(area));
304    }
305
306    static SearchResult searchName(List<SearchResult> results) {
307        return results.stream().filter(
308                x -> OsmPrimitiveType.NODE != x.getOsmId().getType()).iterator().next();
309    }
310
311    static String geocodeArea(String area) throws IOException {
312        // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
313        final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
314        idOffset.put(OsmPrimitiveType.NODE, 0L);
315        idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
316        idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
317        final PrimitiveId osmId = searchName(area).getOsmId();
318        Logging.debug("Area ''{0}'' resolved to {1}", area, osmId);
319        return String.format(Locale.ENGLISH, "area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
320    }
321
322    static String geocodeBbox(String area) throws IOException {
323        Bounds bounds = searchName(area).getBounds();
324        return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
325    }
326
327    static String geocodeCoords(String area) throws IOException {
328        SearchResult result = searchName(area);
329        return result.getLat() + "," + result.getLon();
330    }
331
332    static String geocodeId(String area) throws IOException {
333        PrimitiveId osmId = searchName(area).getOsmId();
334        return String.format(Locale.ENGLISH, "%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
335    }
336
337    @Override
338    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
339                                            boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
340        try {
341            int index = urlStr.indexOf(DATA_PREFIX);
342            // Make an HTTP POST request instead of a simple GET, allows more complex queries
343            return super.getInputStreamRaw(urlStr.substring(0, index),
344                    progressMonitor, reason, uncompressAccordingToContentDisposition,
345                    "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
346        } catch (OsmApiException ex) {
347            final String errorIndicator = "Error</strong>: ";
348            if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
349                final String errorPlusRest = ex.getMessage().split(errorIndicator, -1)[1];
350                if (errorPlusRest != null) {
351                    ex.setErrorHeader(errorPlusRest.split("</", -1)[0].replaceAll(".*::request_read_and_idx::", ""));
352                }
353            }
354            throw ex;
355        }
356    }
357
358    @Override
359    protected void adaptRequest(HttpClient request) {
360        // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
361        final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
362        final int timeout;
363        if (timeoutMatcher.find()) {
364            timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
365        } else {
366            timeout = (int) TimeUnit.MINUTES.toMillis(3);
367        }
368        request.setConnectTimeout(timeout);
369        request.setReadTimeout(timeout);
370    }
371
372    @Override
373    protected String getTaskName() {
374        return tr("Contacting Server...");
375    }
376
377    @Override
378    protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
379        AbstractReader reader = null;
380        Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
381        if (m.matches()) {
382            Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutputFormat.from(m.group(1)));
383            if (readerClass != null) {
384                try {
385                    reader = readerClass.getDeclaredConstructor().newInstance();
386                } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
387                    Logging.error(e);
388                }
389            }
390        }
391        if (reader == null) {
392            reader = new OverpassOsmReader();
393        }
394        return reader.doParseDataSet(source, progressMonitor);
395    }
396
397    @Override
398    public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
399
400        DataSet ds = super.parseOsm(progressMonitor);
401        if (!considerAsFullDownload()) {
402            DataSet noBounds = new DataSet();
403            DataSetMerger dsm = new DataSetMerger(noBounds, ds);
404            dsm.merge(null, false);
405            return dsm.getTargetDataSet();
406        } else {
407            // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
408            if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
409                if (crosses180th) {
410                    Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
411                    DataSource src = new DataSource(bounds, getBaseUrl());
412                    ds.addDataSource(src);
413
414                    bounds = new Bounds(lat1, -180.0, lat2, lon2);
415                    src = new DataSource(bounds, getBaseUrl());
416                    ds.addDataSource(src);
417                } else {
418                    Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
419                    DataSource src = new DataSource(bounds, getBaseUrl());
420                    ds.addDataSource(src);
421                }
422            }
423            return ds;
424        }
425    }
426
427    /**
428     * Fixes Overpass API query to make sure it will be accepted by JOSM.
429     * @param query Overpass query to check
430     * @return fixed query
431     * @since 13335
432     */
433    public static String fixQuery(String query) {
434        return query == null ? query : query
435                .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
436                .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]");
437    }
438
439    @Override
440    public boolean considerAsFullDownload() {
441        return overpassQuery.equals(OverpassDownloadSource.FULL_DOWNLOAD_QUERY);
442    }
443}