001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedInputStream;
007import java.io.BufferedReader;
008import java.io.IOException;
009import java.io.InputStream;
010import java.io.InputStreamReader;
011import java.io.StringReader;
012import java.nio.charset.StandardCharsets;
013import java.util.ArrayList;
014import java.util.Arrays;
015import java.util.List;
016import java.util.Map;
017import java.util.Objects;
018import java.util.Optional;
019import java.util.stream.Collectors;
020
021import javax.json.Json;
022import javax.json.JsonArray;
023import javax.json.JsonNumber;
024import javax.json.JsonObject;
025import javax.json.JsonString;
026import javax.json.JsonValue;
027import javax.json.stream.JsonParser;
028import javax.json.stream.JsonParser.Event;
029import javax.json.stream.JsonParsingException;
030
031import org.openstreetmap.josm.data.coor.EastNorth;
032import org.openstreetmap.josm.data.coor.LatLon;
033import org.openstreetmap.josm.data.osm.BBox;
034import org.openstreetmap.josm.data.osm.DataSet;
035import org.openstreetmap.josm.data.osm.Node;
036import org.openstreetmap.josm.data.osm.OsmPrimitive;
037import org.openstreetmap.josm.data.osm.Relation;
038import org.openstreetmap.josm.data.osm.RelationMember;
039import org.openstreetmap.josm.data.osm.Tag;
040import org.openstreetmap.josm.data.osm.TagCollection;
041import org.openstreetmap.josm.data.osm.TagMap;
042import org.openstreetmap.josm.data.osm.UploadPolicy;
043import org.openstreetmap.josm.data.osm.Way;
044import org.openstreetmap.josm.data.projection.Projection;
045import org.openstreetmap.josm.data.projection.Projections;
046import org.openstreetmap.josm.data.validation.TestError;
047import org.openstreetmap.josm.data.validation.tests.DuplicateWay;
048import org.openstreetmap.josm.gui.conflict.tags.TagConflictResolutionUtil;
049import org.openstreetmap.josm.gui.conflict.tags.TagConflictResolverModel;
050import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
051import org.openstreetmap.josm.gui.progress.ProgressMonitor;
052import org.openstreetmap.josm.tools.CheckParameterUtil;
053import org.openstreetmap.josm.tools.Logging;
054import org.openstreetmap.josm.tools.Utils;
055
056/**
057 * Reader that reads GeoJSON files. See <a href="https://tools.ietf.org/html/rfc7946">RFC7946</a> for more information.
058 * @since 15424
059 */
060public class GeoJSONReader extends AbstractReader {
061
062    private static final String CRS = "crs";
063    private static final String NAME = "name";
064    private static final String LINK = "link";
065    private static final String COORDINATES = "coordinates";
066    private static final String FEATURES = "features";
067    private static final String PROPERTIES = "properties";
068    private static final String GEOMETRY = "geometry";
069    private static final String TYPE = "type";
070    /** The record separator is 0x1E per RFC 7464 */
071    private static final byte RECORD_SEPARATOR_BYTE = 0x1E;
072    private Projection projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84
073
074    GeoJSONReader() {
075        // Restricts visibility
076    }
077
078    private void parse(final JsonParser parser) throws IllegalDataException {
079        while (parser.hasNext()) {
080            Event event = parser.next();
081            if (event == Event.START_OBJECT) {
082                parseRoot(parser.getObject());
083            }
084        }
085        parser.close();
086    }
087
088    private void parseRoot(final JsonObject object) throws IllegalDataException {
089        parseCrs(object.getJsonObject(CRS));
090        switch (Optional.ofNullable(object.getJsonString(TYPE))
091                .orElseThrow(() -> new IllegalDataException("No type")).getString()) {
092            case "FeatureCollection":
093                JsonValue.ValueType valueType = object.get(FEATURES).getValueType();
094                CheckParameterUtil.ensureThat(valueType == JsonValue.ValueType.ARRAY, "features must be ARRAY, but is " + valueType);
095                parseFeatureCollection(object.getJsonArray(FEATURES));
096                break;
097            case "Feature":
098                parseFeature(object);
099                break;
100            case "GeometryCollection":
101                parseGeometryCollection(null, object);
102                break;
103            default:
104                parseGeometry(null, object);
105        }
106    }
107
108    /**
109     * Parse CRS as per https://geojson.org/geojson-spec.html#coordinate-reference-system-objects.
110     * CRS are obsolete in RFC7946 but still allowed for interoperability with older applications.
111     * Only named CRS are supported.
112     *
113     * @param crs CRS JSON object
114     * @throws IllegalDataException in case of error
115     */
116    private void parseCrs(final JsonObject crs) throws IllegalDataException {
117        if (crs != null) {
118            // Inspired by https://github.com/JOSM/geojson/commit/f13ceed4645244612a63581c96e20da802779c56
119            JsonObject properties = crs.getJsonObject("properties");
120            if (properties != null) {
121                switch (crs.getString(TYPE)) {
122                    case NAME:
123                        String crsName = properties.getString(NAME);
124                        if ("urn:ogc:def:crs:OGC:1.3:CRS84".equals(crsName)) {
125                            // https://osgeo-org.atlassian.net/browse/GEOT-1710
126                            crsName = "EPSG:4326";
127                        } else if (crsName.startsWith("urn:ogc:def:crs:EPSG:")) {
128                            crsName = crsName.replace("urn:ogc:def:crs:", "");
129                        }
130                        projection = Optional.ofNullable(Projections.getProjectionByCode(crsName))
131                                .orElse(Projections.getProjectionByCode("EPSG:4326")); // WGS84
132                        break;
133                    case LINK: // Not supported (security risk)
134                    default:
135                        throw new IllegalDataException(crs.toString());
136                }
137            }
138        }
139    }
140
141    private void parseFeatureCollection(final JsonArray features) {
142        for (JsonValue feature : features) {
143            if (feature instanceof JsonObject) {
144                parseFeature((JsonObject) feature);
145            }
146        }
147    }
148
149    private void parseFeature(final JsonObject feature) {
150        JsonValue geometry = feature.get(GEOMETRY);
151        if (geometry != null && geometry.getValueType() == JsonValue.ValueType.OBJECT) {
152            parseGeometry(feature, geometry.asJsonObject());
153        } else {
154            JsonValue properties = feature.get(PROPERTIES);
155            if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) {
156                parseNonGeometryFeature(feature, properties.asJsonObject());
157            } else {
158                Logging.warn(tr("Relation/non-geometry feature without properties found: {0}", feature));
159            }
160        }
161    }
162
163    private void parseNonGeometryFeature(final JsonObject feature, final JsonObject properties) {
164        // get relation type
165        JsonValue type = properties.get(TYPE);
166        if (type == null || properties.getValueType() == JsonValue.ValueType.STRING) {
167            Logging.warn(tr("Relation/non-geometry feature without type found: {0}", feature));
168            return;
169        }
170
171        // create misc. non-geometry feature
172        final Relation relation = new Relation();
173        fillTagsFromFeature(feature, relation);
174        relation.put(TYPE, type.toString());
175        getDataSet().addPrimitive(relation);
176    }
177
178    private void parseGeometryCollection(final JsonObject feature, final JsonObject geometry) {
179        for (JsonValue jsonValue : geometry.getJsonArray("geometries")) {
180            parseGeometry(feature, jsonValue.asJsonObject());
181        }
182    }
183
184    private void parseGeometry(final JsonObject feature, final JsonObject geometry) {
185        if (geometry == null) {
186            parseNullGeometry(feature);
187            return;
188        }
189
190        switch (geometry.getString(TYPE)) {
191            case "Point":
192                parsePoint(feature, geometry.getJsonArray(COORDINATES));
193                break;
194            case "MultiPoint":
195                parseMultiPoint(feature, geometry);
196                break;
197            case "LineString":
198                parseLineString(feature, geometry.getJsonArray(COORDINATES));
199                break;
200            case "MultiLineString":
201                parseMultiLineString(feature, geometry);
202                break;
203            case "Polygon":
204                parsePolygon(feature, geometry.getJsonArray(COORDINATES));
205                break;
206            case "MultiPolygon":
207                parseMultiPolygon(feature, geometry);
208                break;
209            case "GeometryCollection":
210                parseGeometryCollection(feature, geometry);
211                break;
212            default:
213                parseUnknown(geometry);
214        }
215    }
216
217    private LatLon getLatLon(final JsonArray coordinates) {
218        return projection.eastNorth2latlon(new EastNorth(
219                parseCoordinate(coordinates.get(0)),
220                parseCoordinate(coordinates.get(1))));
221    }
222
223    private static double parseCoordinate(JsonValue coordinate) {
224        if (coordinate instanceof JsonString) {
225            return Double.parseDouble(((JsonString) coordinate).getString());
226        } else if (coordinate instanceof JsonNumber) {
227            return ((JsonNumber) coordinate).doubleValue();
228        } else {
229            throw new IllegalArgumentException(Objects.toString(coordinate));
230        }
231    }
232
233    private void parsePoint(final JsonObject feature, final JsonArray coordinates) {
234        fillTagsFromFeature(feature, createNode(getLatLon(coordinates)));
235    }
236
237    private void parseMultiPoint(final JsonObject feature, final JsonObject geometry) {
238        for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
239            parsePoint(feature, coordinate.asJsonArray());
240        }
241    }
242
243    private void parseLineString(final JsonObject feature, final JsonArray coordinates) {
244        if (!coordinates.isEmpty()) {
245            createWay(coordinates, false)
246                .ifPresent(way -> fillTagsFromFeature(feature, way));
247        }
248    }
249
250    private void parseMultiLineString(final JsonObject feature, final JsonObject geometry) {
251        for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
252            parseLineString(feature, coordinate.asJsonArray());
253        }
254    }
255
256    private void parsePolygon(final JsonObject feature, final JsonArray coordinates) {
257        final int size = coordinates.size();
258        if (size == 1) {
259            createWay(coordinates.getJsonArray(0), true)
260                .ifPresent(way -> fillTagsFromFeature(feature, way));
261        } else if (size > 1) {
262            // create multipolygon
263            final Relation multipolygon = new Relation();
264            createWay(coordinates.getJsonArray(0), true)
265                .ifPresent(way -> multipolygon.addMember(new RelationMember("outer", way)));
266
267            for (JsonValue interiorRing : coordinates.subList(1, size)) {
268                createWay(interiorRing.asJsonArray(), true)
269                    .ifPresent(way -> multipolygon.addMember(new RelationMember("inner", way)));
270            }
271
272            fillTagsFromFeature(feature, multipolygon);
273            multipolygon.put(TYPE, "multipolygon");
274            getDataSet().addPrimitive(multipolygon);
275        }
276    }
277
278    private void parseMultiPolygon(final JsonObject feature, final JsonObject geometry) {
279        for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
280            parsePolygon(feature, coordinate.asJsonArray());
281        }
282    }
283
284    private Node createNode(final LatLon latlon) {
285        final List<Node> existingNodes = getDataSet().searchNodes(new BBox(latlon, latlon));
286        if (!existingNodes.isEmpty()) {
287            // reuse existing node, avoid multiple nodes on top of each other
288            return existingNodes.get(0);
289        }
290        final Node node = new Node(latlon);
291        getDataSet().addPrimitive(node);
292        return node;
293    }
294
295    private Optional<Way> createWay(final JsonArray coordinates, final boolean autoClose) {
296        if (coordinates.isEmpty()) {
297            return Optional.empty();
298        }
299
300        final List<LatLon> latlons = coordinates.stream()
301                .map(coordinate -> getLatLon(coordinate.asJsonArray()))
302                .collect(Collectors.toList());
303
304        final int size = latlons.size();
305        final boolean doAutoclose;
306        if (size > 1) {
307            if (latlons.get(0).equals(latlons.get(size - 1))) {
308                doAutoclose = false; // already closed
309            } else {
310                doAutoclose = autoClose;
311            }
312        } else {
313            doAutoclose = false;
314        }
315
316        final Way way = new Way();
317        getDataSet().addPrimitive(way);
318        final List<Node> rawNodes = latlons.stream().map(this::createNode).collect(Collectors.toList());
319        if (doAutoclose) {
320            rawNodes.add(rawNodes.get(0));
321        }
322        // see #19833: remove duplicated references to the same node
323        final List<Node> wayNodes = new ArrayList<>(rawNodes.size());
324        Node last = null;
325        for (Node curr : rawNodes) {
326            if (last != curr)
327                wayNodes.add(curr);
328            last = curr;
329        }
330        way.setNodes(wayNodes);
331
332        return Optional.of(way);
333    }
334
335    /**
336     * Merge existing tags in primitive (if any) with the values given in the GeoJSON feature.
337     * @param feature the GeoJSON feature
338     * @param primitive the OSM primitive
339     */
340    private static void fillTagsFromFeature(final JsonObject feature, final OsmPrimitive primitive) {
341        if (feature != null) {
342            TagCollection featureTags = getTags(feature);
343            primitive.setKeys(new TagMap(primitive.isTagged() ? mergeAllTagValues(primitive, featureTags) : featureTags));
344        }
345    }
346
347    private static TagCollection mergeAllTagValues(final OsmPrimitive primitive, TagCollection featureTags) {
348        TagCollection tags = TagCollection.from(primitive).union(featureTags);
349        TagConflictResolutionUtil.applyAutomaticTagConflictResolution(tags);
350        TagConflictResolutionUtil.normalizeTagCollectionBeforeEditing(tags, Arrays.asList(primitive));
351        TagConflictResolverModel tagModel = new TagConflictResolverModel();
352        tagModel.populate(new TagCollection(tags), tags.getKeysWithMultipleValues());
353        tagModel.actOnDecisions((k, d) -> d.keepAll());
354        return tagModel.getAllResolutions();
355    }
356
357    private static void parseUnknown(final JsonObject object) {
358        Logging.warn(tr("Unknown json object found {0}", object));
359    }
360
361    private static void parseNullGeometry(JsonObject feature) {
362        Logging.warn(tr("Geometry of feature {0} is null", feature));
363    }
364
365    private static TagCollection getTags(final JsonObject feature) {
366        final TagCollection tags = new TagCollection();
367
368        if (feature.containsKey(PROPERTIES) && !feature.isNull(PROPERTIES)) {
369            JsonValue properties = feature.get(PROPERTIES);
370            if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) {
371                for (Map.Entry<String, JsonValue> stringJsonValueEntry : properties.asJsonObject().entrySet()) {
372                    final JsonValue value = stringJsonValueEntry.getValue();
373
374                    if (value instanceof JsonString) {
375                        tags.add(new Tag(stringJsonValueEntry.getKey(), ((JsonString) value).getString()));
376                    } else if (value instanceof JsonObject) {
377                        Logging.warn(
378                            "The GeoJSON contains an object with property '" + stringJsonValueEntry.getKey()
379                                + "' whose value has the unsupported type '" + value.getClass().getSimpleName()
380                                + "'. That key-value pair is ignored!"
381                        );
382                    } else if (value.getValueType() != JsonValue.ValueType.NULL) {
383                        tags.add(new Tag(stringJsonValueEntry.getKey(), value.toString()));
384                    }
385                }
386            }
387        }
388        return tags;
389    }
390
391    /**
392     * Check if the inputstream follows RFC 7464
393     * @param source The source to check (should be at the beginning)
394     * @return {@code true} if the initial character is {@link GeoJSONReader#RECORD_SEPARATOR_BYTE}.
395     */
396    private static boolean isLineDelimited(InputStream source) {
397        source.mark(2);
398        try {
399            int start = source.read();
400            if (RECORD_SEPARATOR_BYTE == start) {
401                return true;
402            }
403            source.reset();
404        } catch (IOException e) {
405            Logging.error(e);
406        }
407        return false;
408    }
409
410    @Override
411    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
412        try (InputStream markSupported = source.markSupported() ? source : new BufferedInputStream(source)) {
413            ds.setUploadPolicy(UploadPolicy.DISCOURAGED);
414            if (isLineDelimited(markSupported)) {
415                try (BufferedReader reader = new BufferedReader(new InputStreamReader(markSupported, StandardCharsets.UTF_8))) {
416                    String line;
417                    String rs = new String(new byte[]{RECORD_SEPARATOR_BYTE}, StandardCharsets.US_ASCII);
418                    while ((line = reader.readLine()) != null) {
419                        line = Utils.strip(line, rs);
420                        try (JsonParser parser = Json.createParser(new StringReader(line))) {
421                            parse(parser);
422                        }
423                    }
424                }
425            } else {
426                try (JsonParser parser = Json.createParser(markSupported)) {
427                    parse(parser);
428                }
429            }
430            mergeEqualMultipolygonWays();
431        } catch (IOException | IllegalArgumentException | JsonParsingException e) {
432            throw new IllegalDataException(e);
433        }
434        return getDataSet();
435    }
436
437    /**
438     * Import may create duplicate ways were one is member of a multipolygon and untagged and the other is tagged.
439     * Try to merge them here.
440     */
441    private void mergeEqualMultipolygonWays() {
442        DuplicateWay test = new DuplicateWay();
443        test.startTest(null);
444        for (Way w: getDataSet().getWays()) {
445            test.visit(w);
446        }
447        test.endTest();
448
449        if (test.getErrors().isEmpty())
450            return;
451
452        for (TestError e : test.getErrors()) {
453            if (e.getPrimitives().size() == 2 && !e.isFixable()) {
454                List<Way> mpWays = new ArrayList<>();
455                Way replacement = null;
456                for (OsmPrimitive p : e.getPrimitives()) {
457                    if (p.isTagged() && p.referrers(Relation.class).count() == 0)
458                        replacement = (Way) p;
459                    else if (p.referrers(Relation.class).anyMatch(Relation::isMultipolygon))
460                        mpWays.add((Way) p);
461                }
462                if (replacement == null && mpWays.size() == 2) {
463                    replacement = mpWays.remove(1);
464                }
465                if (replacement != null && mpWays.size() == 1) {
466                    Way mpWay = mpWays.get(0);
467                    for (Relation r : mpWay.referrers(Relation.class).filter(Relation::isMultipolygon)
468                            .collect(Collectors.toList())) {
469                        for (int i = 0; i < r.getMembersCount(); i++) {
470                            if (r.getMember(i).getMember().equals(mpWay)) {
471                                r.setMember(i, new RelationMember(r.getRole(i), replacement));
472                            }
473                        }
474                    }
475                    mpWay.setDeleted(true);
476                }
477            }
478        }
479        ds.cleanupDeletedPrimitives();
480    }
481
482    /**
483     * Parse the given input source and return the dataset.
484     *
485     * @param source          the source input stream. Must not be null.
486     * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
487     * @return the dataset with the parsed data
488     * @throws IllegalDataException     if an error was found while parsing the data from the source
489     * @throws IllegalArgumentException if source is null
490     */
491    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
492        return new GeoJSONReader().doParseDataSet(source, progressMonitor);
493    }
494}