001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedInputStream; 007import java.io.BufferedReader; 008import java.io.IOException; 009import java.io.InputStream; 010import java.io.InputStreamReader; 011import java.io.StringReader; 012import java.nio.charset.StandardCharsets; 013import java.util.ArrayList; 014import java.util.Arrays; 015import java.util.List; 016import java.util.Map; 017import java.util.Objects; 018import java.util.Optional; 019import java.util.stream.Collectors; 020 021import javax.json.Json; 022import javax.json.JsonArray; 023import javax.json.JsonNumber; 024import javax.json.JsonObject; 025import javax.json.JsonString; 026import javax.json.JsonValue; 027import javax.json.stream.JsonParser; 028import javax.json.stream.JsonParser.Event; 029import javax.json.stream.JsonParsingException; 030 031import org.openstreetmap.josm.data.coor.EastNorth; 032import org.openstreetmap.josm.data.coor.LatLon; 033import org.openstreetmap.josm.data.osm.BBox; 034import org.openstreetmap.josm.data.osm.DataSet; 035import org.openstreetmap.josm.data.osm.Node; 036import org.openstreetmap.josm.data.osm.OsmPrimitive; 037import org.openstreetmap.josm.data.osm.Relation; 038import org.openstreetmap.josm.data.osm.RelationMember; 039import org.openstreetmap.josm.data.osm.Tag; 040import org.openstreetmap.josm.data.osm.TagCollection; 041import org.openstreetmap.josm.data.osm.TagMap; 042import org.openstreetmap.josm.data.osm.UploadPolicy; 043import org.openstreetmap.josm.data.osm.Way; 044import org.openstreetmap.josm.data.projection.Projection; 045import org.openstreetmap.josm.data.projection.Projections; 046import org.openstreetmap.josm.data.validation.TestError; 047import org.openstreetmap.josm.data.validation.tests.DuplicateWay; 048import org.openstreetmap.josm.gui.conflict.tags.TagConflictResolutionUtil; 049import org.openstreetmap.josm.gui.conflict.tags.TagConflictResolverModel; 050import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 051import org.openstreetmap.josm.gui.progress.ProgressMonitor; 052import org.openstreetmap.josm.tools.CheckParameterUtil; 053import org.openstreetmap.josm.tools.Logging; 054import org.openstreetmap.josm.tools.Utils; 055 056/** 057 * Reader that reads GeoJSON files. See <a href="https://tools.ietf.org/html/rfc7946">RFC7946</a> for more information. 058 * @since 15424 059 */ 060public class GeoJSONReader extends AbstractReader { 061 062 private static final String CRS = "crs"; 063 private static final String NAME = "name"; 064 private static final String LINK = "link"; 065 private static final String COORDINATES = "coordinates"; 066 private static final String FEATURES = "features"; 067 private static final String PROPERTIES = "properties"; 068 private static final String GEOMETRY = "geometry"; 069 private static final String TYPE = "type"; 070 /** The record separator is 0x1E per RFC 7464 */ 071 private static final byte RECORD_SEPARATOR_BYTE = 0x1E; 072 private Projection projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84 073 074 GeoJSONReader() { 075 // Restricts visibility 076 } 077 078 private void parse(final JsonParser parser) throws IllegalDataException { 079 while (parser.hasNext()) { 080 Event event = parser.next(); 081 if (event == Event.START_OBJECT) { 082 parseRoot(parser.getObject()); 083 } 084 } 085 parser.close(); 086 } 087 088 private void parseRoot(final JsonObject object) throws IllegalDataException { 089 parseCrs(object.getJsonObject(CRS)); 090 switch (Optional.ofNullable(object.getJsonString(TYPE)) 091 .orElseThrow(() -> new IllegalDataException("No type")).getString()) { 092 case "FeatureCollection": 093 JsonValue.ValueType valueType = object.get(FEATURES).getValueType(); 094 CheckParameterUtil.ensureThat(valueType == JsonValue.ValueType.ARRAY, "features must be ARRAY, but is " + valueType); 095 parseFeatureCollection(object.getJsonArray(FEATURES)); 096 break; 097 case "Feature": 098 parseFeature(object); 099 break; 100 case "GeometryCollection": 101 parseGeometryCollection(null, object); 102 break; 103 default: 104 parseGeometry(null, object); 105 } 106 } 107 108 /** 109 * Parse CRS as per https://geojson.org/geojson-spec.html#coordinate-reference-system-objects. 110 * CRS are obsolete in RFC7946 but still allowed for interoperability with older applications. 111 * Only named CRS are supported. 112 * 113 * @param crs CRS JSON object 114 * @throws IllegalDataException in case of error 115 */ 116 private void parseCrs(final JsonObject crs) throws IllegalDataException { 117 if (crs != null) { 118 // Inspired by https://github.com/JOSM/geojson/commit/f13ceed4645244612a63581c96e20da802779c56 119 JsonObject properties = crs.getJsonObject("properties"); 120 if (properties != null) { 121 switch (crs.getString(TYPE)) { 122 case NAME: 123 String crsName = properties.getString(NAME); 124 if ("urn:ogc:def:crs:OGC:1.3:CRS84".equals(crsName)) { 125 // https://osgeo-org.atlassian.net/browse/GEOT-1710 126 crsName = "EPSG:4326"; 127 } else if (crsName.startsWith("urn:ogc:def:crs:EPSG:")) { 128 crsName = crsName.replace("urn:ogc:def:crs:", ""); 129 } 130 projection = Optional.ofNullable(Projections.getProjectionByCode(crsName)) 131 .orElse(Projections.getProjectionByCode("EPSG:4326")); // WGS84 132 break; 133 case LINK: // Not supported (security risk) 134 default: 135 throw new IllegalDataException(crs.toString()); 136 } 137 } 138 } 139 } 140 141 private void parseFeatureCollection(final JsonArray features) { 142 for (JsonValue feature : features) { 143 if (feature instanceof JsonObject) { 144 parseFeature((JsonObject) feature); 145 } 146 } 147 } 148 149 private void parseFeature(final JsonObject feature) { 150 JsonValue geometry = feature.get(GEOMETRY); 151 if (geometry != null && geometry.getValueType() == JsonValue.ValueType.OBJECT) { 152 parseGeometry(feature, geometry.asJsonObject()); 153 } else { 154 JsonValue properties = feature.get(PROPERTIES); 155 if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) { 156 parseNonGeometryFeature(feature, properties.asJsonObject()); 157 } else { 158 Logging.warn(tr("Relation/non-geometry feature without properties found: {0}", feature)); 159 } 160 } 161 } 162 163 private void parseNonGeometryFeature(final JsonObject feature, final JsonObject properties) { 164 // get relation type 165 JsonValue type = properties.get(TYPE); 166 if (type == null || properties.getValueType() == JsonValue.ValueType.STRING) { 167 Logging.warn(tr("Relation/non-geometry feature without type found: {0}", feature)); 168 return; 169 } 170 171 // create misc. non-geometry feature 172 final Relation relation = new Relation(); 173 fillTagsFromFeature(feature, relation); 174 relation.put(TYPE, type.toString()); 175 getDataSet().addPrimitive(relation); 176 } 177 178 private void parseGeometryCollection(final JsonObject feature, final JsonObject geometry) { 179 for (JsonValue jsonValue : geometry.getJsonArray("geometries")) { 180 parseGeometry(feature, jsonValue.asJsonObject()); 181 } 182 } 183 184 private void parseGeometry(final JsonObject feature, final JsonObject geometry) { 185 if (geometry == null) { 186 parseNullGeometry(feature); 187 return; 188 } 189 190 switch (geometry.getString(TYPE)) { 191 case "Point": 192 parsePoint(feature, geometry.getJsonArray(COORDINATES)); 193 break; 194 case "MultiPoint": 195 parseMultiPoint(feature, geometry); 196 break; 197 case "LineString": 198 parseLineString(feature, geometry.getJsonArray(COORDINATES)); 199 break; 200 case "MultiLineString": 201 parseMultiLineString(feature, geometry); 202 break; 203 case "Polygon": 204 parsePolygon(feature, geometry.getJsonArray(COORDINATES)); 205 break; 206 case "MultiPolygon": 207 parseMultiPolygon(feature, geometry); 208 break; 209 case "GeometryCollection": 210 parseGeometryCollection(feature, geometry); 211 break; 212 default: 213 parseUnknown(geometry); 214 } 215 } 216 217 private LatLon getLatLon(final JsonArray coordinates) { 218 return projection.eastNorth2latlon(new EastNorth( 219 parseCoordinate(coordinates.get(0)), 220 parseCoordinate(coordinates.get(1)))); 221 } 222 223 private static double parseCoordinate(JsonValue coordinate) { 224 if (coordinate instanceof JsonString) { 225 return Double.parseDouble(((JsonString) coordinate).getString()); 226 } else if (coordinate instanceof JsonNumber) { 227 return ((JsonNumber) coordinate).doubleValue(); 228 } else { 229 throw new IllegalArgumentException(Objects.toString(coordinate)); 230 } 231 } 232 233 private void parsePoint(final JsonObject feature, final JsonArray coordinates) { 234 fillTagsFromFeature(feature, createNode(getLatLon(coordinates))); 235 } 236 237 private void parseMultiPoint(final JsonObject feature, final JsonObject geometry) { 238 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) { 239 parsePoint(feature, coordinate.asJsonArray()); 240 } 241 } 242 243 private void parseLineString(final JsonObject feature, final JsonArray coordinates) { 244 if (!coordinates.isEmpty()) { 245 createWay(coordinates, false) 246 .ifPresent(way -> fillTagsFromFeature(feature, way)); 247 } 248 } 249 250 private void parseMultiLineString(final JsonObject feature, final JsonObject geometry) { 251 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) { 252 parseLineString(feature, coordinate.asJsonArray()); 253 } 254 } 255 256 private void parsePolygon(final JsonObject feature, final JsonArray coordinates) { 257 final int size = coordinates.size(); 258 if (size == 1) { 259 createWay(coordinates.getJsonArray(0), true) 260 .ifPresent(way -> fillTagsFromFeature(feature, way)); 261 } else if (size > 1) { 262 // create multipolygon 263 final Relation multipolygon = new Relation(); 264 createWay(coordinates.getJsonArray(0), true) 265 .ifPresent(way -> multipolygon.addMember(new RelationMember("outer", way))); 266 267 for (JsonValue interiorRing : coordinates.subList(1, size)) { 268 createWay(interiorRing.asJsonArray(), true) 269 .ifPresent(way -> multipolygon.addMember(new RelationMember("inner", way))); 270 } 271 272 fillTagsFromFeature(feature, multipolygon); 273 multipolygon.put(TYPE, "multipolygon"); 274 getDataSet().addPrimitive(multipolygon); 275 } 276 } 277 278 private void parseMultiPolygon(final JsonObject feature, final JsonObject geometry) { 279 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) { 280 parsePolygon(feature, coordinate.asJsonArray()); 281 } 282 } 283 284 private Node createNode(final LatLon latlon) { 285 final List<Node> existingNodes = getDataSet().searchNodes(new BBox(latlon, latlon)); 286 if (!existingNodes.isEmpty()) { 287 // reuse existing node, avoid multiple nodes on top of each other 288 return existingNodes.get(0); 289 } 290 final Node node = new Node(latlon); 291 getDataSet().addPrimitive(node); 292 return node; 293 } 294 295 private Optional<Way> createWay(final JsonArray coordinates, final boolean autoClose) { 296 if (coordinates.isEmpty()) { 297 return Optional.empty(); 298 } 299 300 final List<LatLon> latlons = coordinates.stream() 301 .map(coordinate -> getLatLon(coordinate.asJsonArray())) 302 .collect(Collectors.toList()); 303 304 final int size = latlons.size(); 305 final boolean doAutoclose; 306 if (size > 1) { 307 if (latlons.get(0).equals(latlons.get(size - 1))) { 308 doAutoclose = false; // already closed 309 } else { 310 doAutoclose = autoClose; 311 } 312 } else { 313 doAutoclose = false; 314 } 315 316 final Way way = new Way(); 317 getDataSet().addPrimitive(way); 318 final List<Node> rawNodes = latlons.stream().map(this::createNode).collect(Collectors.toList()); 319 if (doAutoclose) { 320 rawNodes.add(rawNodes.get(0)); 321 } 322 // see #19833: remove duplicated references to the same node 323 final List<Node> wayNodes = new ArrayList<>(rawNodes.size()); 324 Node last = null; 325 for (Node curr : rawNodes) { 326 if (last != curr) 327 wayNodes.add(curr); 328 last = curr; 329 } 330 way.setNodes(wayNodes); 331 332 return Optional.of(way); 333 } 334 335 /** 336 * Merge existing tags in primitive (if any) with the values given in the GeoJSON feature. 337 * @param feature the GeoJSON feature 338 * @param primitive the OSM primitive 339 */ 340 private static void fillTagsFromFeature(final JsonObject feature, final OsmPrimitive primitive) { 341 if (feature != null) { 342 TagCollection featureTags = getTags(feature); 343 primitive.setKeys(new TagMap(primitive.isTagged() ? mergeAllTagValues(primitive, featureTags) : featureTags)); 344 } 345 } 346 347 private static TagCollection mergeAllTagValues(final OsmPrimitive primitive, TagCollection featureTags) { 348 TagCollection tags = TagCollection.from(primitive).union(featureTags); 349 TagConflictResolutionUtil.applyAutomaticTagConflictResolution(tags); 350 TagConflictResolutionUtil.normalizeTagCollectionBeforeEditing(tags, Arrays.asList(primitive)); 351 TagConflictResolverModel tagModel = new TagConflictResolverModel(); 352 tagModel.populate(new TagCollection(tags), tags.getKeysWithMultipleValues()); 353 tagModel.actOnDecisions((k, d) -> d.keepAll()); 354 return tagModel.getAllResolutions(); 355 } 356 357 private static void parseUnknown(final JsonObject object) { 358 Logging.warn(tr("Unknown json object found {0}", object)); 359 } 360 361 private static void parseNullGeometry(JsonObject feature) { 362 Logging.warn(tr("Geometry of feature {0} is null", feature)); 363 } 364 365 private static TagCollection getTags(final JsonObject feature) { 366 final TagCollection tags = new TagCollection(); 367 368 if (feature.containsKey(PROPERTIES) && !feature.isNull(PROPERTIES)) { 369 JsonValue properties = feature.get(PROPERTIES); 370 if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) { 371 for (Map.Entry<String, JsonValue> stringJsonValueEntry : properties.asJsonObject().entrySet()) { 372 final JsonValue value = stringJsonValueEntry.getValue(); 373 374 if (value instanceof JsonString) { 375 tags.add(new Tag(stringJsonValueEntry.getKey(), ((JsonString) value).getString())); 376 } else if (value instanceof JsonObject) { 377 Logging.warn( 378 "The GeoJSON contains an object with property '" + stringJsonValueEntry.getKey() 379 + "' whose value has the unsupported type '" + value.getClass().getSimpleName() 380 + "'. That key-value pair is ignored!" 381 ); 382 } else if (value.getValueType() != JsonValue.ValueType.NULL) { 383 tags.add(new Tag(stringJsonValueEntry.getKey(), value.toString())); 384 } 385 } 386 } 387 } 388 return tags; 389 } 390 391 /** 392 * Check if the inputstream follows RFC 7464 393 * @param source The source to check (should be at the beginning) 394 * @return {@code true} if the initial character is {@link GeoJSONReader#RECORD_SEPARATOR_BYTE}. 395 */ 396 private static boolean isLineDelimited(InputStream source) { 397 source.mark(2); 398 try { 399 int start = source.read(); 400 if (RECORD_SEPARATOR_BYTE == start) { 401 return true; 402 } 403 source.reset(); 404 } catch (IOException e) { 405 Logging.error(e); 406 } 407 return false; 408 } 409 410 @Override 411 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 412 try (InputStream markSupported = source.markSupported() ? source : new BufferedInputStream(source)) { 413 ds.setUploadPolicy(UploadPolicy.DISCOURAGED); 414 if (isLineDelimited(markSupported)) { 415 try (BufferedReader reader = new BufferedReader(new InputStreamReader(markSupported, StandardCharsets.UTF_8))) { 416 String line; 417 String rs = new String(new byte[]{RECORD_SEPARATOR_BYTE}, StandardCharsets.US_ASCII); 418 while ((line = reader.readLine()) != null) { 419 line = Utils.strip(line, rs); 420 try (JsonParser parser = Json.createParser(new StringReader(line))) { 421 parse(parser); 422 } 423 } 424 } 425 } else { 426 try (JsonParser parser = Json.createParser(markSupported)) { 427 parse(parser); 428 } 429 } 430 mergeEqualMultipolygonWays(); 431 } catch (IOException | IllegalArgumentException | JsonParsingException e) { 432 throw new IllegalDataException(e); 433 } 434 return getDataSet(); 435 } 436 437 /** 438 * Import may create duplicate ways were one is member of a multipolygon and untagged and the other is tagged. 439 * Try to merge them here. 440 */ 441 private void mergeEqualMultipolygonWays() { 442 DuplicateWay test = new DuplicateWay(); 443 test.startTest(null); 444 for (Way w: getDataSet().getWays()) { 445 test.visit(w); 446 } 447 test.endTest(); 448 449 if (test.getErrors().isEmpty()) 450 return; 451 452 for (TestError e : test.getErrors()) { 453 if (e.getPrimitives().size() == 2 && !e.isFixable()) { 454 List<Way> mpWays = new ArrayList<>(); 455 Way replacement = null; 456 for (OsmPrimitive p : e.getPrimitives()) { 457 if (p.isTagged() && p.referrers(Relation.class).count() == 0) 458 replacement = (Way) p; 459 else if (p.referrers(Relation.class).anyMatch(Relation::isMultipolygon)) 460 mpWays.add((Way) p); 461 } 462 if (replacement == null && mpWays.size() == 2) { 463 replacement = mpWays.remove(1); 464 } 465 if (replacement != null && mpWays.size() == 1) { 466 Way mpWay = mpWays.get(0); 467 for (Relation r : mpWay.referrers(Relation.class).filter(Relation::isMultipolygon) 468 .collect(Collectors.toList())) { 469 for (int i = 0; i < r.getMembersCount(); i++) { 470 if (r.getMember(i).getMember().equals(mpWay)) { 471 r.setMember(i, new RelationMember(r.getRole(i), replacement)); 472 } 473 } 474 } 475 mpWay.setDeleted(true); 476 } 477 } 478 } 479 ds.cleanupDeletedPrimitives(); 480 } 481 482 /** 483 * Parse the given input source and return the dataset. 484 * 485 * @param source the source input stream. Must not be null. 486 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 487 * @return the dataset with the parsed data 488 * @throws IllegalDataException if an error was found while parsing the data from the source 489 * @throws IllegalArgumentException if source is null 490 */ 491 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 492 return new GeoJSONReader().doParseDataSet(source, progressMonitor); 493 } 494}