001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.InputStream; 007import java.util.Arrays; 008import java.util.Collection; 009import java.util.Collections; 010import java.util.Objects; 011import java.util.Set; 012import java.util.TreeSet; 013import java.util.regex.Matcher; 014import java.util.regex.Pattern; 015 016import javax.xml.stream.Location; 017import javax.xml.stream.XMLStreamConstants; 018import javax.xml.stream.XMLStreamException; 019import javax.xml.stream.XMLStreamReader; 020 021import org.openstreetmap.josm.data.osm.Changeset; 022import org.openstreetmap.josm.data.osm.DataSet; 023import org.openstreetmap.josm.data.osm.Node; 024import org.openstreetmap.josm.data.osm.NodeData; 025import org.openstreetmap.josm.data.osm.PrimitiveData; 026import org.openstreetmap.josm.data.osm.Relation; 027import org.openstreetmap.josm.data.osm.RelationData; 028import org.openstreetmap.josm.data.osm.RelationMemberData; 029import org.openstreetmap.josm.data.osm.Tagged; 030import org.openstreetmap.josm.data.osm.Way; 031import org.openstreetmap.josm.data.osm.WayData; 032import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 033import org.openstreetmap.josm.gui.progress.ProgressMonitor; 034import org.openstreetmap.josm.tools.Logging; 035import org.openstreetmap.josm.tools.UncheckedParseException; 036import org.openstreetmap.josm.tools.XmlUtils; 037 038/** 039 * Parser for the Osm API (XML output). Read from an input stream and construct a dataset out of it. 040 * 041 * For each xml element, there is a dedicated method. 042 * The XMLStreamReader cursor points to the start of the element, when the method is 043 * entered, and it must point to the end of the same element, when it is exited. 044 */ 045public class OsmReader extends AbstractReader { 046 047 /** 048 * Options are used to change how the xml data is parsed. 049 * For example, {@link Options#CONVERT_UNKNOWN_TO_TAGS} is used to convert unknown XML attributes to a tag for the object. 050 * @since 16641 051 */ 052 public enum Options { 053 /** 054 * Convert unknown XML attributes to tags 055 */ 056 CONVERT_UNKNOWN_TO_TAGS, 057 /** 058 * Save the original id of an object (currently stored in `current_id`) 059 */ 060 SAVE_ORIGINAL_ID 061 } 062 063 protected XMLStreamReader parser; 064 065 /** The {@link OsmReader.Options} to use when parsing the xml data */ 066 protected final Collection<Options> options; 067 068 private static final Set<String> COMMON_XML_ATTRIBUTES = new TreeSet<>(); 069 070 static { 071 COMMON_XML_ATTRIBUTES.add("id"); 072 COMMON_XML_ATTRIBUTES.add("timestamp"); 073 COMMON_XML_ATTRIBUTES.add("user"); 074 COMMON_XML_ATTRIBUTES.add("uid"); 075 COMMON_XML_ATTRIBUTES.add("visible"); 076 COMMON_XML_ATTRIBUTES.add("version"); 077 COMMON_XML_ATTRIBUTES.add("action"); 078 COMMON_XML_ATTRIBUTES.add("changeset"); 079 COMMON_XML_ATTRIBUTES.add("lat"); 080 COMMON_XML_ATTRIBUTES.add("lon"); 081 } 082 083 /** 084 * constructor (for private and subclasses use only) 085 * 086 * @see #parseDataSet(InputStream, ProgressMonitor) 087 */ 088 protected OsmReader() { 089 this((Options) null); 090 } 091 092 /** 093 * constructor (for private and subclasses use only) 094 * @param options The options to use when reading data 095 * 096 * @see #parseDataSet(InputStream, ProgressMonitor) 097 * @since 16641 098 */ 099 protected OsmReader(Options... options) { 100 // Restricts visibility 101 this.options = options == null ? Collections.emptyList() : Arrays.asList(options); 102 } 103 104 protected void setParser(XMLStreamReader parser) { 105 this.parser = parser; 106 } 107 108 protected void throwException(Throwable th) throws XMLStreamException { 109 throw new XmlStreamParsingException(th.getMessage(), parser.getLocation(), th); 110 } 111 112 protected void throwException(String msg, Throwable th) throws XMLStreamException { 113 throw new XmlStreamParsingException(msg, parser.getLocation(), th); 114 } 115 116 protected void throwException(String msg) throws XMLStreamException { 117 throw new XmlStreamParsingException(msg, parser.getLocation()); 118 } 119 120 protected void parse() throws XMLStreamException { 121 int event = parser.getEventType(); 122 while (true) { 123 if (event == XMLStreamConstants.START_ELEMENT) { 124 parseRoot(); 125 } else if (event == XMLStreamConstants.END_ELEMENT) 126 return; 127 if (parser.hasNext()) { 128 event = parser.next(); 129 } else { 130 break; 131 } 132 } 133 parser.close(); 134 } 135 136 protected void parseRoot() throws XMLStreamException { 137 if ("osm".equals(parser.getLocalName())) { 138 parseOsm(); 139 } else { 140 parseUnknown(); 141 } 142 } 143 144 private void parseOsm() throws XMLStreamException { 145 try { 146 parseVersion(parser.getAttributeValue(null, "version")); 147 parseDownloadPolicy("download", parser.getAttributeValue(null, "download")); 148 parseUploadPolicy("upload", parser.getAttributeValue(null, "upload")); 149 parseLocked(parser.getAttributeValue(null, "locked")); 150 } catch (IllegalDataException e) { 151 throwException(e); 152 } 153 String generator = parser.getAttributeValue(null, "generator"); 154 Long uploadChangesetId = null; 155 if (parser.getAttributeValue(null, "upload-changeset") != null) { 156 uploadChangesetId = getLong("upload-changeset"); 157 } 158 while (parser.hasNext()) { 159 int event = parser.next(); 160 161 if (cancel) { 162 cancel = false; 163 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation()); 164 } 165 166 if (event == XMLStreamConstants.START_ELEMENT) { 167 switch (parser.getLocalName()) { 168 case "bounds": 169 parseBounds(generator); 170 break; 171 case "node": 172 parseNode(); 173 break; 174 case "way": 175 parseWay(); 176 break; 177 case "relation": 178 parseRelation(); 179 break; 180 case "changeset": 181 parseChangeset(uploadChangesetId); 182 break; 183 case "remark": // Used by Overpass API 184 parseRemark(); 185 break; 186 default: 187 parseUnknown(); 188 } 189 } else if (event == XMLStreamConstants.END_ELEMENT) { 190 return; 191 } 192 } 193 } 194 195 private void handleIllegalDataException(IllegalDataException e) throws XMLStreamException { 196 Throwable cause = e.getCause(); 197 if (cause instanceof XMLStreamException) { 198 throw (XMLStreamException) cause; 199 } else { 200 throwException(e); 201 } 202 } 203 204 private void parseRemark() throws XMLStreamException { 205 while (parser.hasNext()) { 206 int event = parser.next(); 207 if (event == XMLStreamConstants.CHARACTERS) { 208 ds.setRemark(parser.getText()); 209 } else if (event == XMLStreamConstants.END_ELEMENT) { 210 return; 211 } 212 } 213 } 214 215 private void parseBounds(String generator) throws XMLStreamException { 216 String minlon = parser.getAttributeValue(null, "minlon"); 217 String minlat = parser.getAttributeValue(null, "minlat"); 218 String maxlon = parser.getAttributeValue(null, "maxlon"); 219 String maxlat = parser.getAttributeValue(null, "maxlat"); 220 String origin = parser.getAttributeValue(null, "origin"); 221 try { 222 parseBounds(generator, minlon, minlat, maxlon, maxlat, origin); 223 } catch (IllegalDataException e) { 224 handleIllegalDataException(e); 225 } 226 jumpToEnd(); 227 } 228 229 protected Node parseNode() throws XMLStreamException { 230 String lat = parser.getAttributeValue(null, "lat"); 231 String lon = parser.getAttributeValue(null, "lon"); 232 try { 233 return parseNode(lat, lon, this::readCommon, this::parseNodeTags); 234 } catch (IllegalDataException e) { 235 handleIllegalDataException(e); 236 } 237 return null; 238 } 239 240 private void parseNodeTags(NodeData n) throws IllegalDataException { 241 try { 242 while (parser.hasNext()) { 243 int event = parser.next(); 244 if (event == XMLStreamConstants.START_ELEMENT) { 245 if ("tag".equals(parser.getLocalName())) { 246 parseTag(n); 247 } else { 248 parseUnknown(); 249 } 250 } else if (event == XMLStreamConstants.END_ELEMENT) { 251 return; 252 } 253 } 254 } catch (XMLStreamException e) { 255 throw new IllegalDataException(e); 256 } 257 } 258 259 protected Way parseWay() throws XMLStreamException { 260 try { 261 return parseWay(this::readCommon, this::parseWayNodesAndTags); 262 } catch (IllegalDataException e) { 263 handleIllegalDataException(e); 264 } 265 return null; 266 } 267 268 private void parseWayNodesAndTags(WayData w, Collection<Long> nodeIds) throws IllegalDataException { 269 try { 270 while (parser.hasNext()) { 271 int event = parser.next(); 272 if (event == XMLStreamConstants.START_ELEMENT) { 273 switch (parser.getLocalName()) { 274 case "nd": 275 nodeIds.add(parseWayNode(w)); 276 break; 277 case "tag": 278 parseTag(w); 279 break; 280 default: 281 parseUnknown(); 282 } 283 } else if (event == XMLStreamConstants.END_ELEMENT) { 284 break; 285 } 286 } 287 } catch (XMLStreamException e) { 288 throw new IllegalDataException(e); 289 } 290 } 291 292 private long parseWayNode(WayData w) throws XMLStreamException { 293 if (parser.getAttributeValue(null, "ref") == null) { 294 throwException( 295 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", Long.toString(w.getUniqueId())) 296 ); 297 } 298 long id = getLong("ref"); 299 if (id == 0) { 300 throwException( 301 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", Long.toString(id)) 302 ); 303 } 304 jumpToEnd(); 305 return id; 306 } 307 308 protected Relation parseRelation() throws XMLStreamException { 309 try { 310 return parseRelation(this::readCommon, this::parseRelationMembersAndTags); 311 } catch (IllegalDataException e) { 312 handleIllegalDataException(e); 313 } 314 return null; 315 } 316 317 private void parseRelationMembersAndTags(RelationData r, Collection<RelationMemberData> members) throws IllegalDataException { 318 try { 319 while (parser.hasNext()) { 320 int event = parser.next(); 321 if (event == XMLStreamConstants.START_ELEMENT) { 322 switch (parser.getLocalName()) { 323 case "member": 324 members.add(parseRelationMember(r)); 325 break; 326 case "tag": 327 parseTag(r); 328 break; 329 default: 330 parseUnknown(); 331 } 332 } else if (event == XMLStreamConstants.END_ELEMENT) { 333 break; 334 } 335 } 336 } catch (XMLStreamException e) { 337 throw new IllegalDataException(e); 338 } 339 } 340 341 private RelationMemberData parseRelationMember(RelationData r) throws XMLStreamException { 342 RelationMemberData result = null; 343 try { 344 String ref = parser.getAttributeValue(null, "ref"); 345 String type = parser.getAttributeValue(null, "type"); 346 String role = parser.getAttributeValue(null, "role"); 347 result = parseRelationMember(r, ref, type, role); 348 jumpToEnd(); 349 } catch (IllegalDataException e) { 350 handleIllegalDataException(e); 351 } 352 return result; 353 } 354 355 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException { 356 357 Long id = null; 358 if (parser.getAttributeValue(null, "id") != null) { 359 id = getLong("id"); 360 } 361 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value 362 if (Objects.equals(id, uploadChangesetId)) { 363 uploadChangeset = new Changeset(id != null ? id.intValue() : 0); 364 while (true) { 365 int event = parser.next(); 366 if (event == XMLStreamConstants.START_ELEMENT) { 367 if ("tag".equals(parser.getLocalName())) { 368 parseTag(uploadChangeset); 369 } else { 370 parseUnknown(); 371 } 372 } else if (event == XMLStreamConstants.END_ELEMENT) 373 return; 374 } 375 } else { 376 jumpToEnd(false); 377 } 378 } 379 380 private void parseTag(Tagged t) throws XMLStreamException { 381 String key = parser.getAttributeValue(null, "k"); 382 String value = parser.getAttributeValue(null, "v"); 383 try { 384 parseTag(t, key, value); 385 } catch (IllegalDataException e) { 386 throwException(e); 387 } 388 jumpToEnd(); 389 } 390 391 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 392 final String element = parser.getLocalName(); 393 if (printWarning && ("note".equals(element) || "meta".equals(element))) { 394 // we know that Overpass API returns those elements 395 Logging.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element)); 396 } else if (printWarning) { 397 Logging.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element)); 398 } 399 while (true) { 400 int event = parser.next(); 401 if (event == XMLStreamConstants.START_ELEMENT) { 402 parseUnknown(false); /* no more warning for inner elements */ 403 } else if (event == XMLStreamConstants.END_ELEMENT) 404 return; 405 } 406 } 407 408 protected void parseUnknown() throws XMLStreamException { 409 parseUnknown(true); 410 } 411 412 /** 413 * When cursor is at the start of an element, moves it to the end tag of that element. 414 * Nested content is skipped. 415 * 416 * This is basically the same code as parseUnknown(), except for the warnings, which 417 * are displayed for inner elements and not at top level. 418 * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met 419 * @throws XMLStreamException if there is an error processing the underlying XML source 420 */ 421 protected final void jumpToEnd(boolean printWarning) throws XMLStreamException { 422 while (true) { 423 int event = parser.next(); 424 if (event == XMLStreamConstants.START_ELEMENT) { 425 parseUnknown(printWarning); 426 } else if (event == XMLStreamConstants.END_ELEMENT) 427 return; 428 } 429 } 430 431 protected final void jumpToEnd() throws XMLStreamException { 432 jumpToEnd(true); 433 } 434 435 /** 436 * Read out the common attributes and put them into current OsmPrimitive. 437 * @param current primitive to update 438 * @throws IllegalDataException if there is an error processing the underlying XML source 439 */ 440 private void readCommon(PrimitiveData current) throws IllegalDataException { 441 try { 442 parseId(current, getLong("id")); 443 parseTimestamp(current, parser.getAttributeValue(null, "timestamp")); 444 parseUser(current, parser.getAttributeValue(null, "user"), parser.getAttributeValue(null, "uid")); 445 parseVisible(current, parser.getAttributeValue(null, "visible")); 446 parseVersion(current, parser.getAttributeValue(null, "version")); 447 parseAction(current, parser.getAttributeValue(null, "action")); 448 parseChangeset(current, parser.getAttributeValue(null, "changeset")); 449 450 if (options.contains(Options.SAVE_ORIGINAL_ID)) { 451 parseTag(current, "current_id", Long.toString(getLong("id"))); 452 } 453 if (options.contains(Options.CONVERT_UNKNOWN_TO_TAGS)) { 454 for (int i = 0; i < parser.getAttributeCount(); i++) { 455 if (!COMMON_XML_ATTRIBUTES.contains(parser.getAttributeLocalName(i))) { 456 parseTag(current, parser.getAttributeLocalName(i), parser.getAttributeValue(i)); 457 } 458 } 459 } 460 } catch (UncheckedParseException | XMLStreamException e) { 461 throw new IllegalDataException(e); 462 } 463 } 464 465 private long getLong(String name) throws XMLStreamException { 466 String value = parser.getAttributeValue(null, name); 467 try { 468 return getLong(name, value); 469 } catch (IllegalDataException e) { 470 throwException(e); 471 } 472 return 0; // should not happen 473 } 474 475 /** 476 * Exception thrown after user cancelation. 477 */ 478 private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException { 479 /** 480 * Constructs a new {@code OsmParsingCanceledException}. 481 * @param msg The error message 482 * @param location The parser location 483 */ 484 OsmParsingCanceledException(String msg, Location location) { 485 super(msg, location); 486 } 487 } 488 489 @Override 490 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 491 return doParseDataSet(source, progressMonitor, ir -> { 492 try { 493 setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir)); 494 parse(); 495 } catch (XmlStreamParsingException | UncheckedParseException e) { 496 throw new IllegalDataException(e.getMessage(), e); 497 } catch (XMLStreamException e) { 498 String msg = e.getMessage(); 499 Pattern p = Pattern.compile("Message: (.+)"); 500 Matcher m = p.matcher(msg); 501 if (m.find()) { 502 msg = m.group(1); 503 } 504 if (e.getLocation() != null) 505 throw new IllegalDataException(tr("Line {0} column {1}: ", 506 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); 507 else 508 throw new IllegalDataException(msg, e); 509 } 510 }); 511 } 512 513 /** 514 * Parse the given input source and return the dataset. 515 * 516 * @param source the source input stream. Must not be null. 517 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 518 * 519 * @return the dataset with the parsed data 520 * @throws IllegalDataException if an error was found while parsing the data from the source 521 * @throws IllegalArgumentException if source is null 522 */ 523 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 524 return parseDataSet(source, progressMonitor, (Options) null); 525 } 526 527 /** 528 * Parse the given input source and return the dataset. 529 * 530 * @param source the source input stream. Must not be null. 531 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 532 * @param options The options to use when parsing the dataset 533 * 534 * @return the dataset with the parsed data 535 * @throws IllegalDataException if an error was found while parsing the data from the source 536 * @throws IllegalArgumentException if source is null 537 * @since 16641 538 */ 539 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor, Options... options) 540 throws IllegalDataException { 541 return new OsmReader(options).doParseDataSet(source, progressMonitor); 542 } 543}