001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.nio.charset.StandardCharsets; 009import java.time.Duration; 010import java.time.LocalDateTime; 011import java.time.Period; 012import java.time.ZoneId; 013import java.time.ZoneOffset; 014import java.time.format.DateTimeParseException; 015import java.util.Arrays; 016import java.util.EnumMap; 017import java.util.List; 018import java.util.Locale; 019import java.util.Map; 020import java.util.NoSuchElementException; 021import java.util.Objects; 022import java.util.concurrent.ConcurrentHashMap; 023import java.util.concurrent.TimeUnit; 024import java.util.regex.Matcher; 025import java.util.regex.Pattern; 026 027import javax.xml.stream.XMLStreamConstants; 028import javax.xml.stream.XMLStreamException; 029 030import org.openstreetmap.josm.data.Bounds; 031import org.openstreetmap.josm.data.DataSource; 032import org.openstreetmap.josm.data.coor.LatLon; 033import org.openstreetmap.josm.data.osm.BBox; 034import org.openstreetmap.josm.data.osm.DataSet; 035import org.openstreetmap.josm.data.osm.DataSetMerger; 036import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 037import org.openstreetmap.josm.data.osm.PrimitiveId; 038import org.openstreetmap.josm.data.preferences.BooleanProperty; 039import org.openstreetmap.josm.data.preferences.ListProperty; 040import org.openstreetmap.josm.data.preferences.StringProperty; 041import org.openstreetmap.josm.gui.download.OverpassDownloadSource; 042import org.openstreetmap.josm.gui.progress.ProgressMonitor; 043import org.openstreetmap.josm.io.NameFinder.SearchResult; 044import org.openstreetmap.josm.tools.HttpClient; 045import org.openstreetmap.josm.tools.Logging; 046import org.openstreetmap.josm.tools.UncheckedParseException; 047import org.openstreetmap.josm.tools.Utils; 048import org.openstreetmap.josm.tools.date.DateUtils; 049 050/** 051 * Read content from an Overpass server. 052 * 053 * @since 8744 054 */ 055public class OverpassDownloadReader extends BoundingBoxDownloader { 056 057 /** 058 * Property for current Overpass server. 059 * @since 12816 060 */ 061 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server", 062 "https://overpass-api.de/api/"); 063 /** 064 * Property for list of known Overpass servers. 065 * @since 12816 066 */ 067 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers", 068 Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/")); 069 /** 070 * Property to determine if Overpass API should be used for multi-fetch download. 071 * @since 12816 072 */ 073 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false); 074 075 private static final String DATA_PREFIX = "?data="; 076 077 static final class OverpassOsmReader extends OsmReader { 078 @Override 079 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 080 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) { 081 final String text = parser.getElementText(); 082 if (text.contains("runtime error")) { 083 throw new XMLStreamException(text); 084 } 085 } 086 super.parseUnknown(printWarning); 087 } 088 } 089 090 static final class OverpassOsmJsonReader extends OsmJsonReader { 091 092 } 093 094 /** 095 * Possible Overpass API output format, with the {@code [out:<directive>]} statement. 096 * @since 11916 097 */ 098 public enum OverpassOutputFormat { 099 /** Default output format: plain OSM XML */ 100 OSM_XML("xml"), 101 /** OSM JSON format (not GeoJson) */ 102 OSM_JSON("json"), 103 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */ 104 CSV("csv"), 105 /** Custom, see https://overpass-api.de/output_formats.html#custom */ 106 CUSTOM("custom"), 107 /** Popup, see https://overpass-api.de/output_formats.html#popup */ 108 POPUP("popup"), 109 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */ 110 PBF("pbf"); 111 112 private final String directive; 113 114 OverpassOutputFormat(String directive) { 115 this.directive = directive; 116 } 117 118 /** 119 * Returns the directive used in {@code [out:<directive>]} statement. 120 * @return the directive used in {@code [out:<directive>]} statement 121 */ 122 public String getDirective() { 123 return directive; 124 } 125 126 /** 127 * Returns the {@code OverpassOutputFormat} matching the given directive. 128 * @param directive directive used in {@code [out:<directive>]} statement 129 * @return {@code OverpassOutputFormat} matching the given directive 130 * @throws IllegalArgumentException in case of invalid directive 131 */ 132 static OverpassOutputFormat from(String directive) { 133 for (OverpassOutputFormat oof : values()) { 134 if (oof.directive.equals(directive)) { 135 return oof; 136 } 137 } 138 throw new IllegalArgumentException(directive); 139 } 140 } 141 142 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL); 143 144 static final Map<OverpassOutputFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>(); 145 146 final String overpassServer; 147 final String overpassQuery; 148 149 /** 150 * Constructs a new {@code OverpassDownloadReader}. 151 * 152 * @param downloadArea The area to download 153 * @param overpassServer The Overpass server to use 154 * @param overpassQuery The Overpass query 155 */ 156 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) { 157 super(downloadArea); 158 setDoAuthenticate(false); 159 this.overpassServer = overpassServer; 160 this.overpassQuery = overpassQuery.trim(); 161 } 162 163 /** 164 * Registers an OSM reader for the given Overpass output format. 165 * @param format Overpass output format 166 * @param readerClass OSM reader class 167 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping 168 */ 169 public static final Class<? extends AbstractReader> registerOverpassOutputFormatReader( 170 OverpassOutputFormat format, Class<? extends AbstractReader> readerClass) { 171 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass)); 172 } 173 174 static { 175 registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_XML, OverpassOsmReader.class); 176 registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_JSON, OverpassOsmJsonReader.class); 177 } 178 179 @Override 180 protected String getBaseUrl() { 181 return overpassServer; 182 } 183 184 @Override 185 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) { 186 if (overpassQuery.isEmpty()) 187 return super.getRequestForBbox(lon1, lat1, lon2, lat2); 188 else { 189 final String query = this.overpassQuery 190 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2)) 191 .replace("{{center}}", center(lon1, lat1, lon2, lat2)); 192 final String expandedOverpassQuery = expandExtendedQueries(query); 193 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery); 194 } 195 } 196 197 /** 198 * Evaluates some features of overpass turbo extended query syntax. 199 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries 200 * @param query unexpanded query 201 * @return expanded query 202 */ 203 static String expandExtendedQueries(String query) { 204 final StringBuffer sb = new StringBuffer(); 205 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query); 206 while (matcher.find()) { 207 try { 208 switch (matcher.group(1)) { 209 case "date": 210 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now(ZoneId.systemDefault()))); 211 break; 212 case "geocodeArea": 213 matcher.appendReplacement(sb, geocodeArea(matcher.group(2))); 214 break; 215 case "geocodeBbox": 216 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2))); 217 break; 218 case "geocodeCoords": 219 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2))); 220 break; 221 case "geocodeId": 222 matcher.appendReplacement(sb, geocodeId(matcher.group(2))); 223 break; 224 default: 225 Logging.warn("Unsupported syntax: " + matcher.group(1)); 226 } 227 } catch (UncheckedParseException | DateTimeParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) { 228 final String msg = tr("Failed to evaluate {0}", matcher.group()); 229 Logging.log(Logging.LEVEL_WARN, msg, ex); 230 matcher.appendReplacement(sb, "// " + msg + "\n"); 231 } 232 } 233 matcher.appendTail(sb); 234 return sb.toString(); 235 } 236 237 static String bbox(double lon1, double lat1, double lon2, double lat2) { 238 return lat1 + "," + lon1 + "," + lat2 + "," + lon2; 239 } 240 241 static String center(double lon1, double lat1, double lon2, double lat2) { 242 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter(); 243 return c.lat()+ "," + c.lon(); 244 } 245 246 static String date(String dateOrHumanDuration, LocalDateTime from) { 247 try { 248 return DateUtils.parseInstant(dateOrHumanDuration).toString(); 249 } catch (UncheckedParseException e) { 250 Logging.trace(e); 251 } 252 return duration(dateOrHumanDuration, from); 253 } 254 255 static String duration(String humanDuration, LocalDateTime from) { 256 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes 257 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "") 258 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W") 259 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S"); 260 Matcher matcher = Pattern.compile( 261 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+ 262 "((?:[0-9]+D)?)" + 263 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration); 264 boolean javaPer = false; 265 boolean javaDur = false; 266 if (matcher.matches()) { 267 javaPer = !Utils.isEmpty(matcher.group(1)); 268 javaDur = !Utils.isEmpty(matcher.group(3)); 269 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2); 270 if (javaDur) { 271 duration += 'T' + matcher.group(3); 272 } 273 } 274 275 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely. 276 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds). 277 Period p = null; 278 Duration d = null; 279 int idx = duration.indexOf('T'); 280 if (javaPer) { 281 p = Period.parse(javaDur ? duration.substring(0, idx) : duration); 282 } 283 if (javaDur) { 284 d = Duration.parse(javaPer ? 'P' + duration.substring(idx) : duration); 285 } else if (!javaPer) { 286 d = Duration.parse(duration); 287 } 288 289 // Now that period and duration are known, compute the correct date/time 290 LocalDateTime dt = from; 291 if (p != null) { 292 dt = dt.minus(p); 293 } 294 if (d != null) { 295 dt = dt.minus(d); 296 } 297 298 // Returns the date/time formatted in ISO 8601 299 return dt.toInstant(ZoneOffset.UTC).toString(); 300 } 301 302 private static SearchResult searchName(String area) throws IOException { 303 return searchName(NameFinder.queryNominatim(area)); 304 } 305 306 static SearchResult searchName(List<SearchResult> results) { 307 return results.stream().filter( 308 x -> OsmPrimitiveType.NODE != x.getOsmId().getType()).iterator().next(); 309 } 310 311 static String geocodeArea(String area) throws IOException { 312 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id 313 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class); 314 idOffset.put(OsmPrimitiveType.NODE, 0L); 315 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L); 316 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L); 317 final PrimitiveId osmId = searchName(area).getOsmId(); 318 Logging.debug("Area ''{0}'' resolved to {1}", area, osmId); 319 return String.format(Locale.ENGLISH, "area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType())); 320 } 321 322 static String geocodeBbox(String area) throws IOException { 323 Bounds bounds = searchName(area).getBounds(); 324 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon(); 325 } 326 327 static String geocodeCoords(String area) throws IOException { 328 SearchResult result = searchName(area); 329 return result.getLat() + "," + result.getLon(); 330 } 331 332 static String geocodeId(String area) throws IOException { 333 PrimitiveId osmId = searchName(area).getOsmId(); 334 return String.format(Locale.ENGLISH, "%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId()); 335 } 336 337 @Override 338 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 339 boolean uncompressAccordingToContentDisposition) throws OsmTransferException { 340 try { 341 int index = urlStr.indexOf(DATA_PREFIX); 342 // Make an HTTP POST request instead of a simple GET, allows more complex queries 343 return super.getInputStreamRaw(urlStr.substring(0, index), 344 progressMonitor, reason, uncompressAccordingToContentDisposition, 345 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8)); 346 } catch (OsmApiException ex) { 347 final String errorIndicator = "Error</strong>: "; 348 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) { 349 final String errorPlusRest = ex.getMessage().split(errorIndicator, -1)[1]; 350 if (errorPlusRest != null) { 351 ex.setErrorHeader(errorPlusRest.split("</", -1)[0].replaceAll(".*::request_read_and_idx::", "")); 352 } 353 } 354 throw ex; 355 } 356 } 357 358 @Override 359 protected void adaptRequest(HttpClient request) { 360 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout 361 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery); 362 final int timeout; 363 if (timeoutMatcher.find()) { 364 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1))); 365 } else { 366 timeout = (int) TimeUnit.MINUTES.toMillis(3); 367 } 368 request.setConnectTimeout(timeout); 369 request.setReadTimeout(timeout); 370 } 371 372 @Override 373 protected String getTaskName() { 374 return tr("Contacting Server..."); 375 } 376 377 @Override 378 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 379 AbstractReader reader = null; 380 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery); 381 if (m.matches()) { 382 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutputFormat.from(m.group(1))); 383 if (readerClass != null) { 384 try { 385 reader = readerClass.getDeclaredConstructor().newInstance(); 386 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) { 387 Logging.error(e); 388 } 389 } 390 } 391 if (reader == null) { 392 reader = new OverpassOsmReader(); 393 } 394 return reader.doParseDataSet(source, progressMonitor); 395 } 396 397 @Override 398 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 399 400 DataSet ds = super.parseOsm(progressMonitor); 401 if (!considerAsFullDownload()) { 402 DataSet noBounds = new DataSet(); 403 DataSetMerger dsm = new DataSetMerger(noBounds, ds); 404 dsm.merge(null, false); 405 return dsm.getTargetDataSet(); 406 } else { 407 // add bounds if necessary (note that Overpass API does not return bounds in the response XML) 408 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) { 409 if (crosses180th) { 410 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0); 411 DataSource src = new DataSource(bounds, getBaseUrl()); 412 ds.addDataSource(src); 413 414 bounds = new Bounds(lat1, -180.0, lat2, lon2); 415 src = new DataSource(bounds, getBaseUrl()); 416 ds.addDataSource(src); 417 } else { 418 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2); 419 DataSource src = new DataSource(bounds, getBaseUrl()); 420 ds.addDataSource(src); 421 } 422 } 423 return ds; 424 } 425 } 426 427 /** 428 * Fixes Overpass API query to make sure it will be accepted by JOSM. 429 * @param query Overpass query to check 430 * @return fixed query 431 * @since 13335 432 */ 433 public static String fixQuery(String query) { 434 return query == null ? query : query 435 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;") 436 .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]"); 437 } 438 439 @Override 440 public boolean considerAsFullDownload() { 441 return overpassQuery.equals(OverpassDownloadSource.FULL_DOWNLOAD_QUERY); 442 } 443}