001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.util.ArrayList; 008import java.util.Arrays; 009import java.util.Collection; 010import java.util.Collections; 011import java.util.HashMap; 012import java.util.HashSet; 013import java.util.List; 014import java.util.Locale; 015import java.util.Map; 016import java.util.Map.Entry; 017import java.util.Objects; 018import java.util.Set; 019import java.util.stream.Collectors; 020import java.util.stream.Stream; 021 022import org.openstreetmap.josm.command.Command; 023import org.openstreetmap.josm.command.DeleteCommand; 024import org.openstreetmap.josm.data.coor.EastNorth; 025import org.openstreetmap.josm.data.coor.LatLon; 026import org.openstreetmap.josm.data.osm.Node; 027import org.openstreetmap.josm.data.osm.OsmPrimitive; 028import org.openstreetmap.josm.data.osm.Relation; 029import org.openstreetmap.josm.data.osm.RelationMember; 030import org.openstreetmap.josm.data.osm.TagMap; 031import org.openstreetmap.josm.data.osm.Way; 032import org.openstreetmap.josm.data.preferences.DoubleProperty; 033import org.openstreetmap.josm.data.validation.Severity; 034import org.openstreetmap.josm.data.validation.Test; 035import org.openstreetmap.josm.data.validation.TestError; 036import org.openstreetmap.josm.tools.Geometry; 037import org.openstreetmap.josm.tools.Logging; 038import org.openstreetmap.josm.tools.Pair; 039import org.openstreetmap.josm.tools.SubclassFilteredCollection; 040import org.openstreetmap.josm.tools.Territories; 041import org.openstreetmap.josm.tools.Utils; 042 043/** 044 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations. 045 * @since 5644 046 */ 047public class Addresses extends Test { 048 049 protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601; 050 protected static final int DUPLICATE_HOUSE_NUMBER = 2602; 051 protected static final int MULTIPLE_STREET_NAMES = 2603; 052 protected static final int MULTIPLE_STREET_RELATIONS = 2604; 053 protected static final int HOUSE_NUMBER_TOO_FAR = 2605; 054 protected static final int OBSOLETE_RELATION = 2606; 055 056 protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0); 057 protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0); 058 059 // CHECKSTYLE.OFF: SingleSpaceSeparator 060 protected static final String ADDR_HOUSE_NUMBER = "addr:housenumber"; 061 protected static final String ADDR_INTERPOLATION = "addr:interpolation"; 062 protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood"; 063 protected static final String ADDR_PLACE = "addr:place"; 064 protected static final String ADDR_STREET = "addr:street"; 065 protected static final String ADDR_SUBURB = "addr:suburb"; 066 protected static final String ADDR_CITY = "addr:city"; 067 protected static final String ADDR_UNIT = "addr:unit"; 068 protected static final String ADDR_FLATS = "addr:flats"; 069 protected static final String ADDR_HOUSE_NAME = "addr:housename"; 070 protected static final String ADDR_POSTCODE = "addr:postcode"; 071 protected static final String ASSOCIATED_STREET = "associatedStreet"; 072 // CHECKSTYLE.ON: SingleSpaceSeparator 073 074 private Map<String, Collection<OsmPrimitive>> knownAddresses; 075 private Set<String> ignoredAddresses; 076 077 /** 078 * Constructor 079 */ 080 public Addresses() { 081 super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations.")); 082 } 083 084 protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) { 085 final List<Relation> list = p.referrers(Relation.class) 086 .filter(r -> r.hasTag("type", ASSOCIATED_STREET)) 087 .collect(Collectors.toList()); 088 if (list.size() > 1) { 089 Severity level; 090 // warning level only if several relations have different names, see #10945 091 final String name = list.get(0).get("name"); 092 if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) { 093 level = Severity.WARNING; 094 } else { 095 level = Severity.OTHER; 096 } 097 List<OsmPrimitive> errorList = new ArrayList<>(list); 098 errorList.add(0, p); 099 errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS) 100 .message(tr("Multiple associatedStreet relations")) 101 .primitives(errorList) 102 .build()); 103 } 104 return list; 105 } 106 107 /** 108 * Checks for house numbers for which the street is unknown. 109 * @param p primitive to test 110 * @return error found, or null 111 */ 112 protected TestError checkHouseNumbersWithoutStreet(OsmPrimitive p) { 113 // Find house number without proper location 114 // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation) 115 if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD) 116 && getAndCheckAssociatedStreets(p).isEmpty() 117 && p.referrers(Way.class).noneMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) { 118 // no street found 119 TestError e = TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET) 120 .message(tr("House number without street")) 121 .primitives(p) 122 .build(); 123 errors.add(e); 124 return e; 125 } 126 return null; 127 } 128 129 static boolean isPOI(OsmPrimitive p) { 130 return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name") || 131 p.hasTag("barrier", "entrance", "gate"); 132 } 133 134 static boolean hasAddress(OsmPrimitive p) { 135 return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE); 136 } 137 138 /** 139 * adds the OsmPrimitive to the address map if it complies to the restrictions 140 * @param p OsmPrimitive that has an address 141 */ 142 private void collectAddress(OsmPrimitive p) { 143 if (!isPOI(p)) { 144 for (String simplifiedAddress : getSimplifiedAddresses(p)) { 145 if (!ignoredAddresses.contains(simplifiedAddress)) { 146 knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p); 147 } 148 } 149 } 150 } 151 152 protected void initAddressMap(OsmPrimitive primitive) { 153 knownAddresses = new HashMap<>(); 154 ignoredAddresses = new HashSet<>(); 155 for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) { 156 if (p instanceof Node && p.hasKey(ADDR_UNIT, ADDR_FLATS)) { 157 for (OsmPrimitive r : p.getReferrers()) { 158 if (hasAddress(r)) { 159 // ignore addresses of buildings that are connected to addr:unit nodes 160 // it's quite reasonable that there are more buildings with this address 161 for (String simplifiedAddress : getSimplifiedAddresses(r)) { 162 if (!ignoredAddresses.contains(simplifiedAddress)) { 163 ignoredAddresses.add(simplifiedAddress); 164 } else { 165 knownAddresses.remove(simplifiedAddress); 166 } 167 } 168 } 169 } 170 } 171 if (hasAddress(p)) { 172 collectAddress(p); 173 } 174 } 175 } 176 177 @Override 178 public void endTest() { 179 knownAddresses = null; 180 ignoredAddresses = null; 181 super.endTest(); 182 } 183 184 protected List<TestError> checkForDuplicate(OsmPrimitive p) { 185 if (knownAddresses == null) { 186 initAddressMap(p); 187 } 188 if (!isPOI(p) && hasAddress(p)) { 189 List<TestError> result = new ArrayList<>(); 190 for (String simplifiedAddress : getSimplifiedAddresses(p)) { 191 if (!ignoredAddresses.contains(simplifiedAddress) && knownAddresses.containsKey(simplifiedAddress)) { 192 double maxDistance = MAX_DUPLICATE_DISTANCE.get(); 193 for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) { 194 if (p == p2) { 195 continue; 196 } 197 Severity severityLevel; 198 String city1 = p.get(ADDR_CITY); 199 String city2 = p2.get(ADDR_CITY); 200 double distance = getDistance(p, p2); 201 if (city1 != null && city2 != null) { 202 if (city1.equals(city2)) { 203 if ((!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE) 204 || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) 205 && (!p.hasKey(ADDR_SUBURB) || !p2.hasKey(ADDR_SUBURB) 206 || p.get(ADDR_SUBURB).equals(p2.get(ADDR_SUBURB)))) { 207 severityLevel = Severity.WARNING; 208 } else { 209 // address including city identical but postcode or suburb differs 210 // most likely perfectly fine 211 severityLevel = Severity.OTHER; 212 } 213 } else { 214 // address differs only by city - notify if very close, otherwise ignore 215 if (distance < maxDistance) { 216 severityLevel = Severity.OTHER; 217 } else { 218 continue; 219 } 220 } 221 } else { 222 // at least one address has no city specified 223 if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE) 224 && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) { 225 // address including postcode identical 226 severityLevel = Severity.WARNING; 227 } else { 228 // city/postcode unclear - warn if very close, otherwise only notify 229 // TODO: get city from surrounding boundaries? 230 if (distance < maxDistance) { 231 severityLevel = Severity.WARNING; 232 } else { 233 severityLevel = Severity.OTHER; 234 } 235 } 236 } 237 result.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER) 238 .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance) 239 .primitives(Arrays.asList(p, p2)).build()); 240 } 241 knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times 242 } 243 } 244 errors.addAll(result); 245 return Collections.unmodifiableList(result); 246 } 247 return Collections.emptyList(); 248 } 249 250 static List<String> getSimplifiedAddresses(OsmPrimitive p) { 251 String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE); 252 // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal 253 return expandHouseNumber(p.get(ADDR_HOUSE_NUMBER)).stream().map(addrHouseNumber -> Utils.strip(Stream.of( 254 simplifiedStreetName.replaceAll("[ -]", ""), 255 addrHouseNumber, 256 p.get(ADDR_HOUSE_NAME), 257 p.get(ADDR_UNIT), 258 p.get(ADDR_FLATS)) 259 .filter(Objects::nonNull) 260 .collect(Collectors.joining(" "))) 261 .toUpperCase(Locale.ENGLISH)).collect(Collectors.toList()); 262 } 263 264 /** 265 * Split addr:housenumber on , and ; (common separators) 266 * 267 * @param houseNumber The housenumber to be split 268 * @return A list of addr:housenumber equivalents 269 */ 270 static List<String> expandHouseNumber(String houseNumber) { 271 return Arrays.asList(houseNumber.split("[,;]", -1)); 272 } 273 274 @Override 275 public void visit(Node n) { 276 checkHouseNumbersWithoutStreet(n); 277 checkForDuplicate(n); 278 } 279 280 @Override 281 public void visit(Way w) { 282 checkHouseNumbersWithoutStreet(w); 283 checkForDuplicate(w); 284 } 285 286 @Override 287 public void visit(Relation r) { 288 checkHouseNumbersWithoutStreet(r); 289 checkForDuplicate(r); 290 if (r.hasTag("type", ASSOCIATED_STREET)) { 291 checkIfObsolete(r); 292 // Used to count occurrences of each house number in order to find duplicates 293 Map<String, List<OsmPrimitive>> map = new HashMap<>(); 294 // Used to detect different street names 295 String relationName = r.get("name"); 296 Set<OsmPrimitive> wrongStreetNames = new HashSet<>(); 297 // Used to check distance 298 Set<OsmPrimitive> houses = new HashSet<>(); 299 Set<Way> street = new HashSet<>(); 300 for (RelationMember m : r.getMembers()) { 301 String role = m.getRole(); 302 OsmPrimitive p = m.getMember(); 303 if ("house".equals(role)) { 304 houses.add(p); 305 String number = p.get(ADDR_HOUSE_NUMBER); 306 if (number != null) { 307 number = number.trim().toUpperCase(Locale.ENGLISH); 308 List<OsmPrimitive> list = map.computeIfAbsent(number, k -> new ArrayList<>()); 309 list.add(p); 310 } 311 if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) { 312 if (wrongStreetNames.isEmpty()) { 313 wrongStreetNames.add(r); 314 } 315 wrongStreetNames.add(p); 316 } 317 } else if ("street".equals(role)) { 318 if (p instanceof Way) { 319 street.add((Way) p); 320 } 321 if (relationName != null && p.hasTagDifferent("name", relationName)) { 322 if (wrongStreetNames.isEmpty()) { 323 wrongStreetNames.add(r); 324 } 325 wrongStreetNames.add(p); 326 } 327 } 328 } 329 // Report duplicate house numbers 330 for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) { 331 List<OsmPrimitive> list = entry.getValue(); 332 if (list.size() > 1) { 333 errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER) 334 .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey()) 335 .primitives(list) 336 .build()); 337 } 338 } 339 // Report wrong street names 340 if (!wrongStreetNames.isEmpty()) { 341 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES) 342 .message(tr("Multiple street names in relation")) 343 .primitives(wrongStreetNames) 344 .build()); 345 } 346 // Report addresses too far away 347 if (!street.isEmpty()) { 348 for (OsmPrimitive house : houses) { 349 if (house.isUsable()) { 350 checkDistance(house, street); 351 } 352 } 353 } 354 } 355 } 356 357 /** 358 * returns rough distance between two OsmPrimitives 359 * @param a primitive a 360 * @param b primitive b 361 * @return distance of center of bounding boxes in meters 362 */ 363 static double getDistance(OsmPrimitive a, OsmPrimitive b) { 364 LatLon centerA = a.getBBox().getCenter(); 365 LatLon centerB = b.getBBox().getCenter(); 366 return (centerA.greatCircleDistance(centerB)); 367 } 368 369 protected void checkDistance(OsmPrimitive house, Collection<Way> street) { 370 EastNorth centroid; 371 if (house instanceof Node) { 372 centroid = ((Node) house).getEastNorth(); 373 } else if (house instanceof Way) { 374 List<Node> nodes = ((Way) house).getNodes(); 375 if (house.hasKey(ADDR_INTERPOLATION)) { 376 for (Node n : nodes) { 377 if (n.hasKey(ADDR_HOUSE_NUMBER)) { 378 checkDistance(n, street); 379 } 380 } 381 return; 382 } 383 centroid = Geometry.getCentroid(nodes); 384 } else { 385 return; // TODO handle multipolygon houses ? 386 } 387 if (centroid == null) return; // fix #8305 388 double maxDistance = MAX_STREET_DISTANCE.get(); 389 boolean hasIncompleteWays = false; 390 for (Way streetPart : street) { 391 for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) { 392 EastNorth p1 = chunk.a.getEastNorth(); 393 EastNorth p2 = chunk.b.getEastNorth(); 394 if (p1 != null && p2 != null) { 395 EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid); 396 if (closest.distance(centroid) <= maxDistance) { 397 return; 398 } 399 } else { 400 Logging.warn("Addresses test skipped chunk "+chunk+" for street part "+streetPart+" because p1 or p2 is null"); 401 } 402 } 403 if (!hasIncompleteWays && streetPart.isIncomplete()) { 404 hasIncompleteWays = true; 405 } 406 } 407 // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314) 408 if (hasIncompleteWays) return; 409 List<OsmPrimitive> errorList = new ArrayList<>(street); 410 errorList.add(0, house); 411 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR) 412 .message(tr("House number too far from street")) 413 .primitives(errorList) 414 .build()); 415 } 416 417 /** 418 * Check if an associatedStreet Relation is obsolete. This test marks only those relations which 419 * are complete and don't contain any information which isn't also tagged on the members. 420 * The strategy is to avoid any false positive. 421 * @param r the relation 422 */ 423 private void checkIfObsolete(Relation r) { 424 if (r.isIncomplete()) 425 return; 426 /** array of country codes for which the test should be performed. For now, only Germany */ 427 String[] countryCodes = {"DE"}; 428 TagMap neededtagsForHouse = new TagMap(); 429 for (Entry<String, String> tag : r.getKeys().entrySet()) { 430 String key = tag.getKey(); 431 if (key.startsWith("name:")) { 432 return; // maybe check if all members have corresponding tags? 433 } else if (key.startsWith("addr:")) { 434 neededtagsForHouse.put(key, tag.getValue()); 435 } else { 436 switch (key) { 437 case "name": 438 case "type": 439 case "source": 440 break; 441 default: 442 // unexpected tag in relation 443 return; 444 } 445 } 446 } 447 448 for (RelationMember m : r.getMembers()) { 449 if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes)) 450 return; 451 452 String role = m.getRole(); 453 if ("".equals(role)) { 454 if (m.isWay() && m.getMember().hasKey("highway")) { 455 role = "street"; 456 } else if (m.getMember().hasTag("building")) 457 role = "house"; 458 } 459 switch (role) { 460 case "house": 461 case "addr:houselink": 462 case "address": 463 if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER)) 464 return; 465 for (Entry<String, String> tag : neededtagsForHouse.entrySet()) { 466 if (!m.getMember().hasTag(tag.getKey(), tag.getValue())) 467 return; 468 } 469 break; 470 case "street": 471 if (!m.getMember().hasTag("name") && r.hasTag("name")) 472 return; 473 break; 474 default: 475 // unknown role: don't create auto-fix 476 return; 477 } 478 } 479 errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION) 480 .message(tr("Relation is obsolete")) 481 .primitives(r) 482 .build()); 483 } 484 485 private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) { 486 if (countryCodes.length == 0) 487 return true; 488 LatLon center = null; 489 490 if (m.isNode()) { 491 center = m.getNode().getCoor(); 492 } else if (m.isWay()) { 493 center = m.getWay().getBBox().getCenter(); 494 } else if (m.isRelation() && m.getRelation().isMultipolygon()) { 495 center = m.getRelation().getBBox().getCenter(); 496 } 497 if (center == null) 498 return false; 499 for (String country : countryCodes) { 500 if (Territories.isIso3166Code(country, center)) 501 return true; 502 } 503 return false; 504 } 505 506 /** 507 * remove obsolete relation. 508 */ 509 @Override 510 public Command fixError(TestError testError) { 511 return new DeleteCommand(testError.getPrimitives()); 512 } 513 514 @Override 515 public boolean isFixable(TestError testError) { 516 if (!(testError.getTester() instanceof Addresses)) 517 return false; 518 return testError.getCode() == OBSOLETE_RELATION; 519 } 520 521}