001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.InputStream;
007import java.util.Arrays;
008import java.util.Collection;
009import java.util.Collections;
010import java.util.Objects;
011import java.util.Set;
012import java.util.TreeSet;
013import java.util.regex.Matcher;
014import java.util.regex.Pattern;
015
016import javax.xml.stream.Location;
017import javax.xml.stream.XMLStreamConstants;
018import javax.xml.stream.XMLStreamException;
019import javax.xml.stream.XMLStreamReader;
020
021import org.openstreetmap.josm.data.osm.Changeset;
022import org.openstreetmap.josm.data.osm.DataSet;
023import org.openstreetmap.josm.data.osm.Node;
024import org.openstreetmap.josm.data.osm.NodeData;
025import org.openstreetmap.josm.data.osm.PrimitiveData;
026import org.openstreetmap.josm.data.osm.Relation;
027import org.openstreetmap.josm.data.osm.RelationData;
028import org.openstreetmap.josm.data.osm.RelationMemberData;
029import org.openstreetmap.josm.data.osm.Tagged;
030import org.openstreetmap.josm.data.osm.Way;
031import org.openstreetmap.josm.data.osm.WayData;
032import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
033import org.openstreetmap.josm.gui.progress.ProgressMonitor;
034import org.openstreetmap.josm.tools.Logging;
035import org.openstreetmap.josm.tools.UncheckedParseException;
036import org.openstreetmap.josm.tools.XmlUtils;
037
038/**
039 * Parser for the Osm API (XML output). Read from an input stream and construct a dataset out of it.
040 *
041 * For each xml element, there is a dedicated method.
042 * The XMLStreamReader cursor points to the start of the element, when the method is
043 * entered, and it must point to the end of the same element, when it is exited.
044 */
045public class OsmReader extends AbstractReader {
046
047    /**
048     * Options are used to change how the xml data is parsed.
049     * For example, {@link Options#CONVERT_UNKNOWN_TO_TAGS} is used to convert unknown XML attributes to a tag for the object.
050     * @since 16641
051     */
052    public enum Options {
053        /**
054         * Convert unknown XML attributes to tags
055         */
056        CONVERT_UNKNOWN_TO_TAGS,
057        /**
058         * Save the original id of an object (currently stored in `current_id`)
059         */
060        SAVE_ORIGINAL_ID
061    }
062
063    protected XMLStreamReader parser;
064
065    /** The {@link OsmReader.Options} to use when parsing the xml data */
066    protected final Collection<Options> options;
067
068    private static final Set<String> COMMON_XML_ATTRIBUTES = new TreeSet<>();
069
070    static {
071        COMMON_XML_ATTRIBUTES.add("id");
072        COMMON_XML_ATTRIBUTES.add("timestamp");
073        COMMON_XML_ATTRIBUTES.add("user");
074        COMMON_XML_ATTRIBUTES.add("uid");
075        COMMON_XML_ATTRIBUTES.add("visible");
076        COMMON_XML_ATTRIBUTES.add("version");
077        COMMON_XML_ATTRIBUTES.add("action");
078        COMMON_XML_ATTRIBUTES.add("changeset");
079        COMMON_XML_ATTRIBUTES.add("lat");
080        COMMON_XML_ATTRIBUTES.add("lon");
081    }
082
083    /**
084     * constructor (for private and subclasses use only)
085     *
086     * @see #parseDataSet(InputStream, ProgressMonitor)
087     */
088    protected OsmReader() {
089        this((Options) null);
090    }
091
092    /**
093     * constructor (for private and subclasses use only)
094     * @param options The options to use when reading data
095     *
096     * @see #parseDataSet(InputStream, ProgressMonitor)
097     * @since 16641
098     */
099    protected OsmReader(Options... options) {
100        // Restricts visibility
101        this.options = options == null ? Collections.emptyList() : Arrays.asList(options);
102    }
103
104    protected void setParser(XMLStreamReader parser) {
105        this.parser = parser;
106    }
107
108    protected void throwException(Throwable th) throws XMLStreamException {
109        throw new XmlStreamParsingException(th.getMessage(), parser.getLocation(), th);
110    }
111
112    protected void throwException(String msg, Throwable th) throws XMLStreamException {
113        throw new XmlStreamParsingException(msg, parser.getLocation(), th);
114    }
115
116    protected void throwException(String msg) throws XMLStreamException {
117        throw new XmlStreamParsingException(msg, parser.getLocation());
118    }
119
120    protected void parse() throws XMLStreamException {
121        int event = parser.getEventType();
122        while (true) {
123            if (event == XMLStreamConstants.START_ELEMENT) {
124                parseRoot();
125            } else if (event == XMLStreamConstants.END_ELEMENT)
126                return;
127            if (parser.hasNext()) {
128                event = parser.next();
129            } else {
130                break;
131            }
132        }
133        parser.close();
134    }
135
136    protected void parseRoot() throws XMLStreamException {
137        if ("osm".equals(parser.getLocalName())) {
138            parseOsm();
139        } else {
140            parseUnknown();
141        }
142    }
143
144    private void parseOsm() throws XMLStreamException {
145        try {
146            parseVersion(parser.getAttributeValue(null, "version"));
147            parseDownloadPolicy("download", parser.getAttributeValue(null, "download"));
148            parseUploadPolicy("upload", parser.getAttributeValue(null, "upload"));
149            parseLocked(parser.getAttributeValue(null, "locked"));
150        } catch (IllegalDataException e) {
151            throwException(e);
152        }
153        String generator = parser.getAttributeValue(null, "generator");
154        Long uploadChangesetId = null;
155        if (parser.getAttributeValue(null, "upload-changeset") != null) {
156            uploadChangesetId = getLong("upload-changeset");
157        }
158        while (parser.hasNext()) {
159            int event = parser.next();
160
161            if (cancel) {
162                cancel = false;
163                throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
164            }
165
166            if (event == XMLStreamConstants.START_ELEMENT) {
167                switch (parser.getLocalName()) {
168                case "bounds":
169                    parseBounds(generator);
170                    break;
171                case "node":
172                    parseNode();
173                    break;
174                case "way":
175                    parseWay();
176                    break;
177                case "relation":
178                    parseRelation();
179                    break;
180                case "changeset":
181                    parseChangeset(uploadChangesetId);
182                    break;
183                case "remark": // Used by Overpass API
184                    parseRemark();
185                    break;
186                default:
187                    parseUnknown();
188                }
189            } else if (event == XMLStreamConstants.END_ELEMENT) {
190                return;
191            }
192        }
193    }
194
195    private void handleIllegalDataException(IllegalDataException e) throws XMLStreamException {
196        Throwable cause = e.getCause();
197        if (cause instanceof XMLStreamException) {
198            throw (XMLStreamException) cause;
199        } else {
200            throwException(e);
201        }
202    }
203
204    private void parseRemark() throws XMLStreamException {
205        while (parser.hasNext()) {
206            int event = parser.next();
207            if (event == XMLStreamConstants.CHARACTERS) {
208                ds.setRemark(parser.getText());
209            } else if (event == XMLStreamConstants.END_ELEMENT) {
210                return;
211            }
212        }
213    }
214
215    private void parseBounds(String generator) throws XMLStreamException {
216        String minlon = parser.getAttributeValue(null, "minlon");
217        String minlat = parser.getAttributeValue(null, "minlat");
218        String maxlon = parser.getAttributeValue(null, "maxlon");
219        String maxlat = parser.getAttributeValue(null, "maxlat");
220        String origin = parser.getAttributeValue(null, "origin");
221        try {
222            parseBounds(generator, minlon, minlat, maxlon, maxlat, origin);
223        } catch (IllegalDataException e) {
224            handleIllegalDataException(e);
225        }
226        jumpToEnd();
227    }
228
229    protected Node parseNode() throws XMLStreamException {
230        String lat = parser.getAttributeValue(null, "lat");
231        String lon = parser.getAttributeValue(null, "lon");
232        try {
233            return parseNode(lat, lon, this::readCommon, this::parseNodeTags);
234        } catch (IllegalDataException e) {
235            handleIllegalDataException(e);
236        }
237        return null;
238    }
239
240    private void parseNodeTags(NodeData n) throws IllegalDataException {
241        try {
242            while (parser.hasNext()) {
243                int event = parser.next();
244                if (event == XMLStreamConstants.START_ELEMENT) {
245                    if ("tag".equals(parser.getLocalName())) {
246                        parseTag(n);
247                    } else {
248                        parseUnknown();
249                    }
250                } else if (event == XMLStreamConstants.END_ELEMENT) {
251                    return;
252                }
253            }
254        } catch (XMLStreamException e) {
255            throw new IllegalDataException(e);
256        }
257    }
258
259    protected Way parseWay() throws XMLStreamException {
260        try {
261            return parseWay(this::readCommon, this::parseWayNodesAndTags);
262        } catch (IllegalDataException e) {
263            handleIllegalDataException(e);
264        }
265        return null;
266    }
267
268    private void parseWayNodesAndTags(WayData w, Collection<Long> nodeIds) throws IllegalDataException {
269        try {
270            while (parser.hasNext()) {
271                int event = parser.next();
272                if (event == XMLStreamConstants.START_ELEMENT) {
273                    switch (parser.getLocalName()) {
274                    case "nd":
275                        nodeIds.add(parseWayNode(w));
276                        break;
277                    case "tag":
278                        parseTag(w);
279                        break;
280                    default:
281                        parseUnknown();
282                    }
283                } else if (event == XMLStreamConstants.END_ELEMENT) {
284                    break;
285                }
286            }
287        } catch (XMLStreamException e) {
288            throw new IllegalDataException(e);
289        }
290    }
291
292    private long parseWayNode(WayData w) throws XMLStreamException {
293        if (parser.getAttributeValue(null, "ref") == null) {
294            throwException(
295                    tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", Long.toString(w.getUniqueId()))
296            );
297        }
298        long id = getLong("ref");
299        if (id == 0) {
300            throwException(
301                    tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", Long.toString(id))
302            );
303        }
304        jumpToEnd();
305        return id;
306    }
307
308    protected Relation parseRelation() throws XMLStreamException {
309        try {
310            return parseRelation(this::readCommon, this::parseRelationMembersAndTags);
311        } catch (IllegalDataException e) {
312            handleIllegalDataException(e);
313        }
314        return null;
315    }
316
317    private void parseRelationMembersAndTags(RelationData r, Collection<RelationMemberData> members) throws IllegalDataException {
318        try {
319            while (parser.hasNext()) {
320                int event = parser.next();
321                if (event == XMLStreamConstants.START_ELEMENT) {
322                    switch (parser.getLocalName()) {
323                    case "member":
324                        members.add(parseRelationMember(r));
325                        break;
326                    case "tag":
327                        parseTag(r);
328                        break;
329                    default:
330                        parseUnknown();
331                    }
332                } else if (event == XMLStreamConstants.END_ELEMENT) {
333                    break;
334                }
335            }
336        } catch (XMLStreamException e) {
337            throw new IllegalDataException(e);
338        }
339    }
340
341    private RelationMemberData parseRelationMember(RelationData r) throws XMLStreamException {
342        RelationMemberData result = null;
343        try {
344            String ref = parser.getAttributeValue(null, "ref");
345            String type = parser.getAttributeValue(null, "type");
346            String role = parser.getAttributeValue(null, "role");
347            result = parseRelationMember(r, ref, type, role);
348            jumpToEnd();
349        } catch (IllegalDataException e) {
350            handleIllegalDataException(e);
351        }
352        return result;
353    }
354
355    private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
356
357        Long id = null;
358        if (parser.getAttributeValue(null, "id") != null) {
359            id = getLong("id");
360        }
361        // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
362        if (Objects.equals(id, uploadChangesetId)) {
363            uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
364            while (true) {
365                int event = parser.next();
366                if (event == XMLStreamConstants.START_ELEMENT) {
367                    if ("tag".equals(parser.getLocalName())) {
368                        parseTag(uploadChangeset);
369                    } else {
370                        parseUnknown();
371                    }
372                } else if (event == XMLStreamConstants.END_ELEMENT)
373                    return;
374            }
375        } else {
376            jumpToEnd(false);
377        }
378    }
379
380    private void parseTag(Tagged t) throws XMLStreamException {
381        String key = parser.getAttributeValue(null, "k");
382        String value = parser.getAttributeValue(null, "v");
383        try {
384            parseTag(t, key, value);
385        } catch (IllegalDataException e) {
386            throwException(e);
387        }
388        jumpToEnd();
389    }
390
391    protected void parseUnknown(boolean printWarning) throws XMLStreamException {
392        final String element = parser.getLocalName();
393        if (printWarning && ("note".equals(element) || "meta".equals(element))) {
394            // we know that Overpass API returns those elements
395            Logging.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
396        } else if (printWarning) {
397            Logging.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
398        }
399        while (true) {
400            int event = parser.next();
401            if (event == XMLStreamConstants.START_ELEMENT) {
402                parseUnknown(false); /* no more warning for inner elements */
403            } else if (event == XMLStreamConstants.END_ELEMENT)
404                return;
405        }
406    }
407
408    protected void parseUnknown() throws XMLStreamException {
409        parseUnknown(true);
410    }
411
412    /**
413     * When cursor is at the start of an element, moves it to the end tag of that element.
414     * Nested content is skipped.
415     *
416     * This is basically the same code as parseUnknown(), except for the warnings, which
417     * are displayed for inner elements and not at top level.
418     * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met
419     * @throws XMLStreamException if there is an error processing the underlying XML source
420     */
421    protected final void jumpToEnd(boolean printWarning) throws XMLStreamException {
422        while (true) {
423            int event = parser.next();
424            if (event == XMLStreamConstants.START_ELEMENT) {
425                parseUnknown(printWarning);
426            } else if (event == XMLStreamConstants.END_ELEMENT)
427                return;
428        }
429    }
430
431    protected final void jumpToEnd() throws XMLStreamException {
432        jumpToEnd(true);
433    }
434
435    /**
436     * Read out the common attributes and put them into current OsmPrimitive.
437     * @param current primitive to update
438     * @throws IllegalDataException if there is an error processing the underlying XML source
439     */
440    private void readCommon(PrimitiveData current) throws IllegalDataException {
441        try {
442            parseId(current, getLong("id"));
443            parseTimestamp(current, parser.getAttributeValue(null, "timestamp"));
444            parseUser(current, parser.getAttributeValue(null, "user"), parser.getAttributeValue(null, "uid"));
445            parseVisible(current, parser.getAttributeValue(null, "visible"));
446            parseVersion(current, parser.getAttributeValue(null, "version"));
447            parseAction(current, parser.getAttributeValue(null, "action"));
448            parseChangeset(current, parser.getAttributeValue(null, "changeset"));
449
450            if (options.contains(Options.SAVE_ORIGINAL_ID)) {
451                parseTag(current, "current_id", Long.toString(getLong("id")));
452            }
453            if (options.contains(Options.CONVERT_UNKNOWN_TO_TAGS)) {
454                for (int i = 0; i < parser.getAttributeCount(); i++) {
455                    if (!COMMON_XML_ATTRIBUTES.contains(parser.getAttributeLocalName(i))) {
456                        parseTag(current, parser.getAttributeLocalName(i), parser.getAttributeValue(i));
457                    }
458                }
459            }
460        } catch (UncheckedParseException | XMLStreamException e) {
461            throw new IllegalDataException(e);
462        }
463    }
464
465    private long getLong(String name) throws XMLStreamException {
466        String value = parser.getAttributeValue(null, name);
467        try {
468            return getLong(name, value);
469        } catch (IllegalDataException e) {
470            throwException(e);
471        }
472        return 0; // should not happen
473    }
474
475    /**
476     * Exception thrown after user cancelation.
477     */
478    private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException {
479        /**
480         * Constructs a new {@code OsmParsingCanceledException}.
481         * @param msg The error message
482         * @param location The parser location
483         */
484        OsmParsingCanceledException(String msg, Location location) {
485            super(msg, location);
486        }
487    }
488
489    @Override
490    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
491        return doParseDataSet(source, progressMonitor, ir -> {
492            try {
493                setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir));
494                parse();
495            } catch (XmlStreamParsingException | UncheckedParseException e) {
496                throw new IllegalDataException(e.getMessage(), e);
497            } catch (XMLStreamException e) {
498                String msg = e.getMessage();
499                Pattern p = Pattern.compile("Message: (.+)");
500                Matcher m = p.matcher(msg);
501                if (m.find()) {
502                    msg = m.group(1);
503                }
504                if (e.getLocation() != null)
505                    throw new IllegalDataException(tr("Line {0} column {1}: ",
506                            e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
507                else
508                    throw new IllegalDataException(msg, e);
509            }
510        });
511    }
512
513    /**
514     * Parse the given input source and return the dataset.
515     *
516     * @param source the source input stream. Must not be null.
517     * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
518     *
519     * @return the dataset with the parsed data
520     * @throws IllegalDataException if an error was found while parsing the data from the source
521     * @throws IllegalArgumentException if source is null
522     */
523    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
524        return parseDataSet(source, progressMonitor, (Options) null);
525    }
526
527    /**
528     * Parse the given input source and return the dataset.
529     *
530     * @param source the source input stream. Must not be null.
531     * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
532     * @param options The options to use when parsing the dataset
533     *
534     * @return the dataset with the parsed data
535     * @throws IllegalDataException if an error was found while parsing the data from the source
536     * @throws IllegalArgumentException if source is null
537     * @since 16641
538     */
539    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor, Options... options)
540            throws IllegalDataException {
541        return new OsmReader(options).doParseDataSet(source, progressMonitor);
542    }
543}