001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006import static org.openstreetmap.josm.tools.I18n.trn; 007 008import java.awt.GridBagConstraints; 009import java.awt.event.ActionListener; 010import java.io.BufferedReader; 011import java.io.IOException; 012import java.lang.Character.UnicodeBlock; 013import java.util.ArrayList; 014import java.util.Arrays; 015import java.util.Collection; 016import java.util.Collections; 017import java.util.EnumSet; 018import java.util.HashMap; 019import java.util.HashSet; 020import java.util.Iterator; 021import java.util.LinkedHashMap; 022import java.util.LinkedHashSet; 023import java.util.List; 024import java.util.Locale; 025import java.util.Map; 026import java.util.Map.Entry; 027import java.util.OptionalInt; 028import java.util.Set; 029import java.util.regex.Pattern; 030import java.util.stream.Collectors; 031 032import javax.swing.JCheckBox; 033import javax.swing.JLabel; 034import javax.swing.JPanel; 035 036import org.openstreetmap.josm.command.ChangePropertyCommand; 037import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 038import org.openstreetmap.josm.command.Command; 039import org.openstreetmap.josm.command.SequenceCommand; 040import org.openstreetmap.josm.data.osm.AbstractPrimitive; 041import org.openstreetmap.josm.data.osm.DataSet; 042import org.openstreetmap.josm.data.osm.OsmPrimitive; 043import org.openstreetmap.josm.data.osm.OsmUtils; 044import org.openstreetmap.josm.data.osm.Relation; 045import org.openstreetmap.josm.data.osm.RelationMember; 046import org.openstreetmap.josm.data.osm.Tag; 047import org.openstreetmap.josm.data.osm.TagMap; 048import org.openstreetmap.josm.data.osm.Tagged; 049import org.openstreetmap.josm.data.osm.Way; 050import org.openstreetmap.josm.data.osm.visitor.MergeSourceBuildingVisitor; 051import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 052import org.openstreetmap.josm.data.validation.OsmValidator; 053import org.openstreetmap.josm.data.validation.Severity; 054import org.openstreetmap.josm.data.validation.Test.TagTest; 055import org.openstreetmap.josm.data.validation.TestError; 056import org.openstreetmap.josm.data.validation.util.Entities; 057import org.openstreetmap.josm.gui.progress.ProgressMonitor; 058import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset; 059import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem; 060import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetListener; 061import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetType; 062import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets; 063import org.openstreetmap.josm.gui.tagging.presets.items.Check; 064import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup; 065import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem; 066import org.openstreetmap.josm.gui.widgets.EditableList; 067import org.openstreetmap.josm.io.CachedFile; 068import org.openstreetmap.josm.spi.preferences.Config; 069import org.openstreetmap.josm.tools.GBC; 070import org.openstreetmap.josm.tools.Logging; 071import org.openstreetmap.josm.tools.MultiMap; 072import org.openstreetmap.josm.tools.Utils; 073 074/** 075 * Check for misspelled or wrong tags 076 * 077 * @author frsantos 078 * @since 3669 079 */ 080public class TagChecker extends TagTest implements TaggingPresetListener { 081 082 /** The config file of ignored tags */ 083 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg"; 084 /** The config file of dictionary words */ 085 public static final String SPELL_FILE = "resource://data/validator/words.cfg"; 086 087 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */ 088 private static final Map<String, String> harmonizedKeys = new HashMap<>(); 089 /** The spell check preset values which are not stored in TaggingPresets */ 090 private static volatile HashSet<String> additionalPresetsValueData; 091 /** often used tags which are not in presets */ 092 private static final MultiMap<String, String> oftenUsedTags = new MultiMap<>(); 093 private static final Map<TaggingPreset, List<TaggingPresetItem>> presetIndex = new LinkedHashMap<>(); 094 095 private static final Pattern UNWANTED_NON_PRINTING_CONTROL_CHARACTERS = Pattern.compile( 096 "[\\x00-\\x09\\x0B\\x0C\\x0E-\\x1F\\x7F\\u200e-\\u200f\\u202a-\\u202e]"); 097 098 /** The TagChecker data */ 099 private static final List<String> ignoreDataStartsWith = new ArrayList<>(); 100 private static final Set<String> ignoreDataEquals = new HashSet<>(); 101 private static final List<String> ignoreDataEndsWith = new ArrayList<>(); 102 private static final List<Tag> ignoreDataTag = new ArrayList<>(); 103 /** tag keys that have only numerical values in the presets */ 104 private static final Set<String> ignoreForLevenshtein = new HashSet<>(); 105 106 /** tag keys that are allowed to be the same on a multipolygon and an outer way */ 107 private static final Set<String> ignoreForOuterMPSameTagCheck = new HashSet<>(); 108 109 /** The preferences prefix */ 110 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName(); 111 112 MapCSSTagChecker deprecatedChecker; 113 114 /** 115 * The preference key to check values 116 */ 117 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues"; 118 /** 119 * The preference key to check keys 120 */ 121 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys"; 122 /** 123 * The preference key to enable complex checks 124 */ 125 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex"; 126 /** 127 * The preference key to search for fixme tags 128 */ 129 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes"; 130 /** 131 * The preference key to check presets 132 */ 133 public static final String PREF_CHECK_PRESETS_TYPES = PREFIX + ".checkPresetsTypes"; 134 135 /** 136 * The preference key for source files 137 * @see #DEFAULT_SOURCES 138 */ 139 public static final String PREF_SOURCES = PREFIX + ".source"; 140 141 private static final String BEFORE_UPLOAD = "BeforeUpload"; 142 /** 143 * The preference key to check keys - used before upload 144 */ 145 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + BEFORE_UPLOAD; 146 /** 147 * The preference key to check values - used before upload 148 */ 149 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + BEFORE_UPLOAD; 150 /** 151 * The preference key to run complex tests - used before upload 152 */ 153 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + BEFORE_UPLOAD; 154 /** 155 * The preference key to search for fixmes - used before upload 156 */ 157 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + BEFORE_UPLOAD; 158 /** 159 * The preference key to search for presets - used before upload 160 */ 161 public static final String PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD = PREF_CHECK_PRESETS_TYPES + BEFORE_UPLOAD; 162 163 /** 164 * The preference key for the list of tag keys that are allowed to be the same on a multipolygon and an outer way 165 */ 166 public static final String PREF_KEYS_IGNORE_OUTER_MP_SAME_TAG = PREFIX + ".ignore-keys-outer-mp-same-tag"; 167 168 private static final int MAX_LEVENSHTEIN_DISTANCE = 2; 169 170 protected boolean includeOtherSeverity; 171 172 protected boolean checkKeys; 173 protected boolean checkValues; 174 /** Was used for special configuration file, might be used to disable value spell checker. */ 175 protected boolean checkComplex; 176 protected boolean checkFixmes; 177 protected boolean checkPresetsTypes; 178 179 protected JCheckBox prefCheckKeys; 180 protected JCheckBox prefCheckValues; 181 protected JCheckBox prefCheckComplex; 182 protected JCheckBox prefCheckFixmes; 183 protected JCheckBox prefCheckPresetsTypes; 184 185 protected JCheckBox prefCheckKeysBeforeUpload; 186 protected JCheckBox prefCheckValuesBeforeUpload; 187 protected JCheckBox prefCheckComplexBeforeUpload; 188 protected JCheckBox prefCheckFixmesBeforeUpload; 189 protected JCheckBox prefCheckPresetsTypesBeforeUpload; 190 191 // CHECKSTYLE.OFF: SingleSpaceSeparator 192 protected static final int EMPTY_VALUES = 1200; 193 protected static final int INVALID_KEY = 1201; 194 protected static final int INVALID_VALUE = 1202; 195 protected static final int FIXME = 1203; 196 protected static final int INVALID_SPACE = 1204; 197 protected static final int INVALID_KEY_SPACE = 1205; 198 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */ 199 protected static final int LONG_VALUE = 1208; 200 protected static final int LONG_KEY = 1209; 201 protected static final int LOW_CHAR_VALUE = 1210; 202 protected static final int LOW_CHAR_KEY = 1211; 203 protected static final int MISSPELLED_VALUE = 1212; 204 protected static final int MISSPELLED_KEY = 1213; 205 protected static final int MULTIPLE_SPACES = 1214; 206 protected static final int MISSPELLED_VALUE_NO_FIX = 1215; 207 protected static final int UNUSUAL_UNICODE_CHAR_VALUE = 1216; 208 protected static final int INVALID_PRESETS_TYPE = 1217; 209 protected static final int MULTIPOLYGON_NO_AREA = 1218; 210 protected static final int MULTIPOLYGON_INCOMPLETE = 1219; 211 protected static final int MULTIPOLYGON_MAYBE_NO_AREA = 1220; 212 protected static final int MULTIPOLYGON_SAME_TAG_ON_OUTER = 1221; 213 // CHECKSTYLE.ON: SingleSpaceSeparator 214 215 protected EditableList sourcesList; 216 217 private static final List<String> DEFAULT_SOURCES = Arrays.asList(IGNORE_FILE, SPELL_FILE); 218 219 /** 220 * Constructor 221 */ 222 public TagChecker() { 223 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values.")); 224 } 225 226 @Override 227 public void initialize() throws IOException { 228 TaggingPresets.addListener(this); 229 initializeData(); 230 initializePresets(); 231 analysePresets(); 232 } 233 234 /** 235 * Add presets that contain only numerical values to the ignore list 236 */ 237 private static void analysePresets() { 238 for (String key : TaggingPresets.getPresetKeys()) { 239 if (isKeyIgnored(key)) 240 continue; 241 Set<String> values = TaggingPresets.getPresetValues(key); 242 boolean allNumerical = !Utils.isEmpty(values) 243 && values.stream().allMatch(TagChecker::isNum); 244 if (allNumerical) { 245 ignoreForLevenshtein.add(key); 246 } 247 } 248 } 249 250 /** 251 * Reads the spell-check file into a HashMap. 252 * The data file is a list of words, beginning with +/-. If it starts with +, 253 * the word is valid, but if it starts with -, the word should be replaced 254 * by the nearest + word before this. 255 * 256 * @throws IOException if any I/O error occurs 257 */ 258 private static void initializeData() throws IOException { 259 ignoreDataStartsWith.clear(); 260 ignoreDataEquals.clear(); 261 ignoreDataEndsWith.clear(); 262 ignoreDataTag.clear(); 263 harmonizedKeys.clear(); 264 ignoreForLevenshtein.clear(); 265 oftenUsedTags.clear(); 266 presetIndex.clear(); 267 ignoreForOuterMPSameTagCheck.clear(); 268 269 StringBuilder errorSources = new StringBuilder(); 270 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) { 271 try ( 272 CachedFile cf = new CachedFile(source); 273 BufferedReader reader = cf.getContentReader() 274 ) { 275 String okValue = null; 276 boolean tagcheckerfile = false; 277 boolean ignorefile = false; 278 boolean isFirstLine = true; 279 String line; 280 while ((line = reader.readLine()) != null) { 281 if (line.isEmpty()) { 282 // ignore 283 } else if (line.startsWith("#")) { 284 if (line.startsWith("# JOSM TagChecker")) { 285 tagcheckerfile = true; 286 Logging.error(tr("Ignoring {0}. Support was dropped", source)); 287 } else 288 if (line.startsWith("# JOSM IgnoreTags")) { 289 ignorefile = true; 290 if (!DEFAULT_SOURCES.contains(source)) { 291 Logging.info(tr("Adding {0} to ignore tags", source)); 292 } 293 } 294 } else if (ignorefile) { 295 parseIgnoreFileLine(source, line); 296 } else if (tagcheckerfile) { 297 // ignore 298 } else if (line.charAt(0) == '+') { 299 okValue = line.substring(1); 300 } else if (line.charAt(0) == '-' && okValue != null) { 301 String hk = harmonizeKey(line.substring(1)); 302 if (!okValue.equals(hk) && harmonizedKeys.put(hk, okValue) != null && Logging.isDebugEnabled()) { 303 Logging.debug("Line was ignored: " + line); 304 } 305 } else { 306 Logging.error(tr("Invalid spellcheck line: {0}", line)); 307 } 308 if (isFirstLine) { 309 isFirstLine = false; 310 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) { 311 Logging.info(tr("Adding {0} to spellchecker", source)); 312 } 313 } 314 } 315 } catch (IOException e) { 316 Logging.error(e); 317 errorSources.append(source).append('\n'); 318 } 319 } 320 321 if (errorSources.length() > 0) 322 throw new IOException(trn( 323 "Could not access data file:\n{0}", 324 "Could not access data files:\n{0}", errorSources.length(), errorSources)); 325 } 326 327 /** 328 * Parse a line found in a configuration file 329 * @param source name of configuration file 330 * @param line the line to parse 331 */ 332 private static void parseIgnoreFileLine(String source, String line) { 333 line = line.trim(); 334 if (line.length() < 4) { 335 return; 336 } 337 try { 338 String key = line.substring(0, 2); 339 line = line.substring(2); 340 341 switch (key) { 342 case "S:": 343 ignoreDataStartsWith.add(line); 344 break; 345 case "E:": 346 ignoreDataEquals.add(line); 347 addToKeyDictionary(line); 348 break; 349 case "F:": 350 ignoreDataEndsWith.add(line); 351 break; 352 case "K:": 353 Tag tag = Tag.ofString(line); 354 ignoreDataTag.add(tag); 355 oftenUsedTags.put(tag.getKey(), tag.getValue()); 356 addToKeyDictionary(tag.getKey()); 357 break; 358 default: 359 if (!key.startsWith(";")) { 360 Logging.warn("Unsupported TagChecker key: " + key); 361 } 362 } 363 } catch (IllegalArgumentException e) { 364 Logging.error("Invalid line in {0} : {1}", source, e.getMessage()); 365 Logging.trace(e); 366 } 367 } 368 369 private static void addToKeyDictionary(String key) { 370 if (key != null) { 371 String hk = harmonizeKey(key); 372 if (!key.equals(hk)) { 373 harmonizedKeys.put(hk, key); 374 } 375 } 376 } 377 378 /** 379 * Reads the presets data. 380 * 381 */ 382 public static void initializePresets() { 383 384 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true)) 385 return; 386 387 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets(); 388 if (!presets.isEmpty()) { 389 initAdditionalPresetsValueData(); 390 for (TaggingPreset p : presets) { 391 List<TaggingPresetItem> minData = new ArrayList<>(); 392 for (TaggingPresetItem i : p.data) { 393 if (i instanceof KeyedItem) { 394 if (!"none".equals(((KeyedItem) i).match)) 395 minData.add(i); 396 addPresetValue((KeyedItem) i); 397 } else if (i instanceof CheckGroup) { 398 for (Check c : ((CheckGroup) i).checks) { 399 addPresetValue(c); 400 } 401 } 402 } 403 if (!minData.isEmpty()) { 404 presetIndex .put(p, minData); 405 } 406 } 407 } 408 } 409 410 private static void initAdditionalPresetsValueData() { 411 additionalPresetsValueData = new HashSet<>(); 412 additionalPresetsValueData.addAll(AbstractPrimitive.getUninterestingKeys()); 413 additionalPresetsValueData.addAll(Config.getPref().getList( 414 ValidatorPrefHelper.PREFIX + ".knownkeys", 415 Arrays.asList("is_in", "int_ref", "fixme", "population"))); 416 } 417 418 private static void addPresetValue(KeyedItem ky) { 419 if (ky.key != null && ky.getValues() != null) { 420 addToKeyDictionary(ky.key); 421 } 422 } 423 424 /** 425 * Checks given string (key or value) if it contains unwanted non-printing control characters (either ASCII or Unicode bidi characters) 426 * @param s string to check 427 * @return {@code true} if {@code s} contains non-printing control characters 428 */ 429 static boolean containsUnwantedNonPrintingControlCharacter(String s) { 430 return !Utils.isEmpty(s) && ( 431 isJoiningChar(s.charAt(0)) || 432 isJoiningChar(s.charAt(s.length() - 1)) || 433 s.chars().anyMatch(c -> (isAsciiControlChar(c) && !isNewLineChar(c)) || isBidiControlChar(c)) 434 ); 435 } 436 437 private static boolean isAsciiControlChar(int c) { 438 return c < 0x20 || c == 0x7F; 439 } 440 441 private static boolean isNewLineChar(int c) { 442 return c == 0x0a || c == 0x0d; 443 } 444 445 private static boolean isJoiningChar(int c) { 446 return c == 0x200c || c == 0x200d; // ZWNJ, ZWJ 447 } 448 449 private static boolean isBidiControlChar(int c) { 450 /* check for range 0x200e to 0x200f (LRM, RLM) or 451 0x202a to 0x202e (LRE, RLE, PDF, LRO, RLO) */ 452 return (c >= 0x200e && c <= 0x200f) || (c >= 0x202a && c <= 0x202e); 453 } 454 455 static String removeUnwantedNonPrintingControlCharacters(String s) { 456 // Remove all unwanted characters 457 String result = UNWANTED_NON_PRINTING_CONTROL_CHARACTERS.matcher(s).replaceAll(""); 458 // Remove joining characters located at the beginning of the string 459 while (!result.isEmpty() && isJoiningChar(result.charAt(0))) { 460 result = result.substring(1); 461 } 462 // Remove joining characters located at the end of the string 463 while (!result.isEmpty() && isJoiningChar(result.charAt(result.length() - 1))) { 464 result = result.substring(0, result.length() - 1); 465 } 466 return result; 467 } 468 469 static boolean containsUnusualUnicodeCharacter(String key, String value) { 470 return getUnusualUnicodeCharacter(key, value).isPresent(); 471 } 472 473 static OptionalInt getUnusualUnicodeCharacter(String key, String value) { 474 return value == null 475 ? OptionalInt.empty() 476 : value.chars().filter(c -> isUnusualUnicodeBlock(key, c)).findFirst(); 477 } 478 479 /** 480 * Detects highly suspicious Unicode characters that have been seen in OSM database. 481 * @param key tag key 482 * @param c current character code point 483 * @return {@code true} if the current unicode block is very unusual for the given key 484 */ 485 private static boolean isUnusualUnicodeBlock(String key, int c) { 486 UnicodeBlock b = UnicodeBlock.of(c); 487 return isUnusualPhoneticUse(key, b, c) || isUnusualBmpUse(b) || isUnusualSmpUse(b); 488 } 489 490 private static boolean isAllowedPhoneticCharacter(String key, int c) { 491 // CHECKSTYLE.OFF: BooleanExpressionComplexity 492 return c == 0x0259 || c == 0x018F // U+0259 is paired with the capital letter U+018F in Azeri, see #18740 493 || c == 0x0254 || c == 0x0186 // U+0254 is paired with the capital letter U+0186 in several African languages, see #18740 494 || c == 0x0257 || c == 0x018A // "ɗ/Ɗ" (U+0257/U+018A), see #19760 495 || c == 0x025B || c == 0x0190 // U+025B is paired with the capital letter U+0190 in several African languages, see #18740 496 || c == 0x0263 || c == 0x0194 // "ɣ/Ɣ" (U+0263/U+0194), see #18740 497 || c == 0x0268 || c == 0x0197 // "ɨ/Ɨ" (U+0268/U+0197), see #18740 498 || c == 0x0269 || c == 0x0196 // "ɩ/Ɩ" (U+0269/U+0196), see #20437 499 || c == 0x0272 || c == 0x019D // "ɲ/Ɲ" (U+0272/U+019D), see #18740 500 || c == 0x0273 || c == 0x019E // "ŋ/Ŋ" (U+0273/U+019E), see #18740 501 || c == 0x0142 || c == 0x0294 // see #20754 502 || (key.endsWith("ref") && 0x1D2C <= c && c <= 0x1D42); // allow uppercase superscript latin characters in *ref tags 503 } 504 505 private static boolean isUnusualPhoneticUse(String key, UnicodeBlock b, int c) { 506 return !isAllowedPhoneticCharacter(key, c) 507 && (b == UnicodeBlock.IPA_EXTENSIONS // U+0250..U+02AF 508 || b == UnicodeBlock.PHONETIC_EXTENSIONS // U+1D00..U+1D7F 509 || b == UnicodeBlock.PHONETIC_EXTENSIONS_SUPPLEMENT) // U+1D80..U+1DBF 510 && !key.endsWith(":pronunciation"); 511 } 512 513 private static boolean isUnusualBmpUse(UnicodeBlock b) { 514 return b == UnicodeBlock.COMBINING_MARKS_FOR_SYMBOLS // U+20D0..U+20FF 515 || b == UnicodeBlock.MATHEMATICAL_OPERATORS // U+2200..U+22FF 516 || b == UnicodeBlock.ENCLOSED_ALPHANUMERICS // U+2460..U+24FF 517 || b == UnicodeBlock.BOX_DRAWING // U+2500..U+257F 518 || b == UnicodeBlock.GEOMETRIC_SHAPES // U+25A0..U+25FF 519 || b == UnicodeBlock.DINGBATS // U+2700..U+27BF 520 || b == UnicodeBlock.MISCELLANEOUS_SYMBOLS_AND_ARROWS // U+2B00..U+2BFF 521 || b == UnicodeBlock.GLAGOLITIC // U+2C00..U+2C5F 522 || b == UnicodeBlock.HANGUL_COMPATIBILITY_JAMO // U+3130..U+318F 523 || b == UnicodeBlock.ENCLOSED_CJK_LETTERS_AND_MONTHS // U+3200..U+32FF 524 || b == UnicodeBlock.LATIN_EXTENDED_D // U+A720..U+A7FF 525 || b == UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS // U+F900..U+FAFF 526 || b == UnicodeBlock.ALPHABETIC_PRESENTATION_FORMS // U+FB00..U+FB4F 527 || b == UnicodeBlock.VARIATION_SELECTORS // U+FE00..U+FE0F 528 || b == UnicodeBlock.SPECIALS; // U+FFF0..U+FFFF 529 // CHECKSTYLE.ON: BooleanExpressionComplexity 530 } 531 532 private static boolean isUnusualSmpUse(UnicodeBlock b) { 533 // UnicodeBlock.SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS is only defined in Java 9+ 534 return b == UnicodeBlock.MUSICAL_SYMBOLS // U+1D100..U+1D1FF 535 || b == UnicodeBlock.ENCLOSED_ALPHANUMERIC_SUPPLEMENT // U+1F100..U+1F1FF 536 || b == UnicodeBlock.EMOTICONS // U+1F600..U+1F64F 537 || b == UnicodeBlock.TRANSPORT_AND_MAP_SYMBOLS; // U+1F680..U+1F6FF 538 } 539 540 /** 541 * Get set of preset values for the given key. 542 * @param key the key 543 * @return null if key is not in presets or in additionalPresetsValueData, 544 * else a set which might be empty. 545 */ 546 private static Set<String> getPresetValues(String key) { 547 if (TaggingPresets.isKeyInPresets(key)) { 548 return TaggingPresets.getPresetValues(key); 549 } 550 if (additionalPresetsValueData.contains(key)) 551 return Collections.emptySet(); 552 // null means key is not known 553 return null; 554 } 555 556 /** 557 * Determines if the given key is in internal presets. 558 * @param key key 559 * @return {@code true} if the given key is in internal presets 560 * @since 9023 561 * @deprecated Use {@link TaggingPresets#isKeyInPresets(String)} instead 562 */ 563 @Deprecated 564 public static boolean isKeyInPresets(String key) { 565 return TaggingPresets.isKeyInPresets(key); 566 } 567 568 /** 569 * Determines if the given tag is in internal presets. 570 * @param key key 571 * @param value value 572 * @return {@code true} if the given tag is in internal presets 573 * @since 9023 574 */ 575 public static boolean isTagInPresets(String key, String value) { 576 final Set<String> values = getPresetValues(key); 577 return values != null && values.contains(value); 578 } 579 580 /** 581 * Returns the list of ignored tags. 582 * @return the list of ignored tags 583 * @since 9023 584 */ 585 public static List<Tag> getIgnoredTags() { 586 return new ArrayList<>(ignoreDataTag); 587 } 588 589 /** 590 * Determines if the given tag key is ignored for checks "key/tag not in presets". 591 * @param key key 592 * @return true if the given key is ignored 593 */ 594 private static boolean isKeyIgnored(String key) { 595 return ignoreDataEquals.contains(key) 596 || ignoreDataStartsWith.stream().anyMatch(key::startsWith) 597 || ignoreDataEndsWith.stream().anyMatch(key::endsWith); 598 } 599 600 /** 601 * Determines if the given tag is ignored for checks "key/tag not in presets". 602 * @param key key 603 * @param value value 604 * @return {@code true} if the given tag is ignored 605 * @since 9023 606 */ 607 public static boolean isTagIgnored(String key, String value) { 608 if (isKeyIgnored(key)) 609 return true; 610 final Set<String> values = getPresetValues(key); 611 if (values != null && values.isEmpty()) 612 return true; 613 if (!isTagInPresets(key, value)) { 614 return ignoreDataTag.stream() 615 .anyMatch(a -> key.equals(a.getKey()) && value.equals(a.getValue())); 616 } 617 return false; 618 } 619 620 /** 621 * Checks the primitive tags 622 * @param p The primitive to check 623 */ 624 @Override 625 public void check(OsmPrimitive p) { 626 if (!p.isTagged()) 627 return; 628 629 // Just a collection to know if a primitive has been already marked with error 630 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>(); 631 632 for (Entry<String, String> prop : p.getKeys().entrySet()) { 633 String s = marktr("Tag ''{0}'' invalid."); 634 String key = prop.getKey(); 635 String value = prop.getValue(); 636 637 if (checkKeys) { 638 checkSingleTagKeySimple(withErrors, p, s, key); 639 } 640 if (checkValues) { 641 checkSingleTagValueSimple(withErrors, p, s, key, value); 642 checkSingleTagComplex(withErrors, p, key, value); 643 } 644 if (checkFixmes && key != null && !Utils.isEmpty(value) && isFixme(key, value) && !withErrors.contains(p, "FIXME")) { 645 errors.add(TestError.builder(this, Severity.OTHER, FIXME) 646 .message(tr("fixme")) 647 .primitives(p) 648 .build()); 649 withErrors.put(p, "FIXME"); 650 } 651 } 652 653 if (p instanceof Relation && p.hasTag("type", "multipolygon")) { 654 checkMultipolygonTags(p); 655 } 656 657 if (checkPresetsTypes) { 658 TagMap tags = p.getKeys(); 659 TaggingPresetType presetType = TaggingPresetType.forPrimitive(p); 660 EnumSet<TaggingPresetType> presetTypes = EnumSet.of(presetType); 661 662 Collection<TaggingPreset> matchingPresets = presetIndex.entrySet().stream() 663 .filter(e -> TaggingPresetItem.matches(e.getValue(), tags)) 664 .map(Entry::getKey) 665 .collect(Collectors.toCollection(LinkedHashSet::new)); 666 Collection<TaggingPreset> matchingPresetsOK = matchingPresets.stream().filter( 667 tp -> tp.typeMatches(presetTypes)).collect(Collectors.toList()); 668 Collection<TaggingPreset> matchingPresetsKO = matchingPresets.stream().filter( 669 tp -> !tp.typeMatches(presetTypes)).collect(Collectors.toList()); 670 671 for (TaggingPreset tp : matchingPresetsKO) { 672 // Potential error, unless matching tags are all known by a supported preset 673 Map<String, String> matchingTags = tp.data.stream() 674 .filter(i -> Boolean.TRUE.equals(i.matches(tags))) 675 .filter(i -> i instanceof KeyedItem).map(i -> ((KeyedItem) i).key) 676 .collect(Collectors.toMap(k -> k, tags::get)); 677 if (matchingPresetsOK.stream().noneMatch( 678 tp2 -> matchingTags.entrySet().stream().allMatch( 679 e -> tp2.data.stream().anyMatch( 680 i -> i instanceof KeyedItem && ((KeyedItem) i).key.equals(e.getKey()))))) { 681 errors.add(TestError.builder(this, Severity.OTHER, INVALID_PRESETS_TYPE) 682 .message(tr("Object type not in preset"), 683 marktr("Object type {0} is not supported by tagging preset: {1}"), 684 tr(presetType.getName()), tp.getLocaleName()) 685 .primitives(p) 686 .build()); 687 } 688 } 689 } 690 } 691 692 private static final Collection<String> NO_AREA_KEYS = Arrays.asList("name", "area", "ref", "access", "operator"); 693 694 private void checkMultipolygonTags(OsmPrimitive p) { 695 if (p.isAnnotated() || p.keys() 696 .anyMatch(k -> k.matches("^(abandoned|construction|demolished|disused|planned|razed|removed|was).*"))) 697 return; 698 699 checkOuterWaysOfRelation((Relation) p); 700 701 if (hasAcceptedPrimaryTagForMultipolygon(p)) 702 return; 703 TestError.Builder builder = null; 704 if (p.hasKey("surface")) { 705 // accept often used tag surface=* as area tag 706 builder = TestError.builder(this, Severity.OTHER, MULTIPOLYGON_INCOMPLETE) 707 .message(tr("Multipolygon tags"), marktr("only {0} tag"), "surface"); 708 } else { 709 Map<String, String> filteredTags = p.getInterestingTags(); 710 filteredTags.remove("type"); 711 NO_AREA_KEYS.forEach(filteredTags::remove); 712 filteredTags.keySet().removeIf(key -> !key.matches("[a-z0-9:_]+")); 713 714 if (filteredTags.isEmpty()) { 715 builder = TestError.builder(this, Severity.ERROR, MULTIPOLYGON_NO_AREA) 716 .message(tr("Multipolygon tags"), marktr("tag describing the area is missing"), new Object()); 717 718 } 719 } 720 if (builder == null) { 721 // multipolygon has either no area tag or a rarely used one 722 builder = TestError.builder(this, Severity.WARNING, MULTIPOLYGON_MAYBE_NO_AREA) 723 .message(tr("Multipolygon tags"), marktr("tag describing the area might be missing"), new Object()); 724 } 725 errors.add(builder.primitives(p).build()); 726 } 727 728 /** 729 * Check if an outer way of the relation has the same tag as the relation. 730 * @param rel the relation 731 */ 732 private void checkOuterWaysOfRelation(Relation rel) { 733 for (Entry<String, String> tag : rel.getInterestingTags().entrySet()) { 734 if (ignoreForOuterMPSameTagCheck.contains(tag.getKey())) 735 continue; 736 737 Set<Way> sameOuters = rel.getMembers().stream() 738 .filter(rm -> rm.isWay() && rm.getWay().isArea() && "outer".equals(rm.getRole()) 739 && tag.getValue().equals(rm.getWay().get(tag.getKey()))) 740 .map(RelationMember::getWay).collect(Collectors.toSet()); 741 if (!sameOuters.isEmpty()) { 742 List<OsmPrimitive> primitives = new ArrayList<>(sameOuters.size() + 1); 743 primitives.add(rel); 744 primitives.addAll(sameOuters); 745 Way w = new Way(); 746 w.put(tag.getKey(), tag.getValue()); 747 if (hasAcceptedPrimaryTagForMultipolygon(w)) { 748 errors.add(TestError.builder(this, Severity.WARNING, MULTIPOLYGON_SAME_TAG_ON_OUTER) 749 .message(tr("Multipolygon outer way repeats major tag of relation"), 750 marktr("Same tag:''{0}''=''{1}''"), tag.getKey(), tag.getValue()) 751 .primitives(primitives) 752 .build()); 753 } else { 754 errors.add(TestError.builder(this, Severity.OTHER, MULTIPOLYGON_SAME_TAG_ON_OUTER) 755 .message(tr("Multipolygon outer way repeats tag of relation"), 756 marktr("Same tag:''{0}''=''{1}''"), tag.getKey(), tag.getValue()) 757 .primitives(primitives) 758 .build()); 759 } 760 } 761 } 762 } 763 764 /** 765 * Check if a multipolygon has a main tag that describes the type of area. Accepts also some deprecated tags and typos. 766 * @param p the multipolygon 767 * @return true if the multipolygon has a main tag that (likely) describes the type of area. 768 */ 769 private static boolean hasAcceptedPrimaryTagForMultipolygon(OsmPrimitive p) { 770 if (p.hasKey("landuse", "amenity", "building", "building:part", "area:highway", "shop", "place", "boundary", 771 "landform", "piste:type", "sport", "golf", "landcover", "aeroway", "office", "healthcare", "craft", "room") 772 || p.hasTagDifferent("natural", "tree", "peek", "saddle", "tree_row") 773 || p.hasTagDifferent("man_made", "survey_point", "mast", "flagpole", "manhole", "watertap") 774 || p.hasTagDifferent("highway", "crossing", "bus_stop", "turning_circle", "street_lamp", 775 "traffic_signals", "stop", "milestone", "mini_roundabout", "motorway_junction", "passing_place", 776 "speed_camera", "traffic_mirror", "trailhead", "turning_circle", "turning_loop", "toll_gantry") 777 || p.hasTagDifferent("tourism", "attraction", "artwork") 778 || p.hasTagDifferent("leisure", "picnic_table", "slipway", "firepit") 779 || p.hasTagDifferent("historic", "wayside_cross", "milestone")) 780 return true; 781 if (p.hasTag("barrier", "hedge", "retaining_wall") 782 || p.hasTag("public_transport", "platform", "station") 783 || p.hasTag("railway", "platform") 784 || p.hasTag("waterway", "riverbank", "dam", "rapids", "dock", "boatyard", "fuel") 785 || p.hasTag("indoor", "corridor", "room", "area") 786 || p.hasTag("power", "substation", "generator", "plant", "switchgear", "converter", "sub_station") 787 || p.hasTag("seamark:type", "harbour", "fairway", "anchorage", "landmark", "berth", "harbour_basin", 788 "separation_zone") 789 || (p.get("seamark:type") != null && p.get("seamark:type").matches(".*\\_(area|zone)$"))) 790 return true; 791 return p.hasTag("harbour", OsmUtils.TRUE_VALUE) 792 || p.hasTag("flood_prone", OsmUtils.TRUE_VALUE) 793 || p.hasTag("bridge", OsmUtils.TRUE_VALUE) 794 || p.hasTag("ruins", OsmUtils.TRUE_VALUE) 795 || p.hasTag("junction", OsmUtils.TRUE_VALUE); 796 } 797 798 private void checkSingleTagValueSimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key, String value) { 799 if (!checkValues || value == null) 800 return; 801 if (containsUnwantedNonPrintingControlCharacter(value) && !withErrors.contains(p, "ICV")) { 802 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE) 803 .message(tr("Tag value contains non-printing (usually invisible) character"), s, key) 804 .primitives(p) 805 .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(value))) 806 .build()); 807 withErrors.put(p, "ICV"); 808 } 809 final OptionalInt unusualUnicodeCharacter = getUnusualUnicodeCharacter(key, value); 810 if (unusualUnicodeCharacter.isPresent() && !withErrors.contains(p, "UUCV")) { 811 final String codepoint = String.format(Locale.ROOT, "U+%04X", unusualUnicodeCharacter.getAsInt()); 812 errors.add(TestError.builder(this, Severity.WARNING, UNUSUAL_UNICODE_CHAR_VALUE) 813 .message(tr("Tag value contains unusual Unicode character {0}", codepoint), s, key) 814 .primitives(p) 815 .build()); 816 withErrors.put(p, "UUCV"); 817 } 818 if ((value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) { 819 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE) 820 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key) 821 .primitives(p) 822 .build()); 823 withErrors.put(p, "LV"); 824 } 825 if (value.trim().isEmpty() && !withErrors.contains(p, "EV")) { 826 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES) 827 .message(tr("Tags with empty values"), s, key) 828 .primitives(p) 829 .build()); 830 withErrors.put(p, "EV"); 831 } 832 final String errTypeSpace = "SPACE"; 833 if ((value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, errTypeSpace)) { 834 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE) 835 .message(tr("Property values start or end with white space"), s, key) 836 .primitives(p) 837 .build()); 838 withErrors.put(p, errTypeSpace); 839 } 840 if (value.contains(" ") && !withErrors.contains(p, errTypeSpace)) { 841 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES) 842 .message(tr("Property values contain multiple white spaces"), s, key) 843 .primitives(p) 844 .build()); 845 withErrors.put(p, errTypeSpace); 846 } 847 if (includeOtherSeverity && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) { 848 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML) 849 .message(tr("Property values contain HTML entity"), s, key) 850 .primitives(p) 851 .build()); 852 withErrors.put(p, "HTML"); 853 } 854 } 855 856 private void checkSingleTagKeySimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key) { 857 if (!checkKeys || key == null) 858 return; 859 if (containsUnwantedNonPrintingControlCharacter(key) && !withErrors.contains(p, "ICK")) { 860 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY) 861 .message(tr("Tag key contains non-printing character"), s, key) 862 .primitives(p) 863 .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(key))) 864 .build()); 865 withErrors.put(p, "ICK"); 866 } 867 if (key.length() > Tagged.MAX_TAG_LENGTH && !withErrors.contains(p, "LK")) { 868 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY) 869 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key) 870 .primitives(p) 871 .build()); 872 withErrors.put(p, "LK"); 873 } 874 if (key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) { 875 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE) 876 .message(tr("Invalid white space in property key"), s, key) 877 .primitives(p) 878 .build()); 879 withErrors.put(p, "IPK"); 880 } 881 } 882 883 private void checkSingleTagComplex(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key, String value) { 884 if (!checkValues || key == null || Utils.isEmpty(value)) 885 return; 886 if (additionalPresetsValueData != null && !isTagIgnored(key, value)) { 887 if (!TaggingPresets.isKeyInPresets(key)) { 888 spellCheckKey(withErrors, p, key); 889 } else if (!isTagInPresets(key, value)) { 890 if (oftenUsedTags.contains(key, value)) { 891 // tag is quite often used but not in presets 892 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 893 .message(tr("Presets do not contain property value"), 894 marktr("Value ''{0}'' for key ''{1}'' not in presets, but is known."), value, key) 895 .primitives(p) 896 .build()); 897 withErrors.put(p, "UPV"); 898 } else { 899 tryGuess(p, key, value, withErrors); 900 } 901 } 902 } 903 } 904 905 private void spellCheckKey(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key) { 906 String prettifiedKey = harmonizeKey(key); 907 String fixedKey; 908 if (ignoreDataEquals.contains(prettifiedKey)) { 909 fixedKey = prettifiedKey; 910 } else { 911 fixedKey = TaggingPresets.isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey); 912 } 913 if (fixedKey == null && ignoreDataTag.stream().anyMatch(a -> a.getKey().equals(prettifiedKey))) { 914 fixedKey = prettifiedKey; 915 } 916 917 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) { 918 final String proposedKey = fixedKey; 919 // misspelled preset key 920 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY) 921 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, proposedKey) 922 .primitives(p); 923 if (p.hasKey(fixedKey)) { 924 errors.add(error.build()); 925 } else { 926 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, proposedKey)).build()); 927 } 928 withErrors.put(p, "WPK"); 929 } else if (includeOtherSeverity) { 930 errors.add(TestError.builder(this, Severity.OTHER, INVALID_KEY) 931 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key) 932 .primitives(p) 933 .build()); 934 withErrors.put(p, "UPK"); 935 } 936 } 937 938 private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) { 939 // try to fix common typos and check again if value is still unknown 940 final String harmonizedValue = harmonizeValue(value); 941 if (Utils.isEmpty(harmonizedValue)) 942 return; 943 String fixedValue; 944 List<Set<String>> sets = new ArrayList<>(); 945 Set<String> presetValues = getPresetValues(key); 946 if (presetValues != null) 947 sets.add(presetValues); 948 Set<String> usedValues = oftenUsedTags.get(key); 949 if (usedValues != null) 950 sets.add(usedValues); 951 fixedValue = sets.stream().anyMatch(possibleValues -> possibleValues.contains(harmonizedValue)) 952 ? harmonizedValue : null; 953 if (fixedValue == null && !ignoreForLevenshtein.contains(key)) { 954 int maxPresetValueLen = 0; 955 List<String> fixVals = new ArrayList<>(); 956 // use Levenshtein distance to find typical typos 957 int minDist = MAX_LEVENSHTEIN_DISTANCE + 1; 958 for (Set<String> possibleValues: sets) { 959 for (String possibleVal : possibleValues) { 960 if (possibleVal.isEmpty()) 961 continue; 962 maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length()); 963 if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) { 964 // don't suggest fix value when given value is short and lengths are too different 965 // for example surface=u would result in surface=mud 966 continue; 967 } 968 int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue); 969 if (dist >= harmonizedValue.length()) { 970 // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'. 971 continue; 972 } 973 if (dist < minDist) { 974 minDist = dist; 975 fixVals.clear(); 976 fixVals.add(possibleVal); 977 } else if (dist == minDist) { 978 fixVals.add(possibleVal); 979 } 980 } 981 } 982 if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE 983 && !fixVals.isEmpty() 984 && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) { 985 filterDeprecatedTags(p, key, fixVals); 986 if (!fixVals.isEmpty()) { 987 if (fixVals.size() < 2) { 988 fixedValue = fixVals.get(0); 989 } else { 990 Collections.sort(fixVals); 991 // misspelled preset value with multiple good alternatives 992 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX) 993 .message(tr("Unknown property value"), 994 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"), 995 value, key, fixVals) 996 .primitives(p).build()); 997 withErrors.put(p, "WPV"); 998 return; 999 } 1000 } 1001 } 1002 } 1003 if (fixedValue != null && !fixedValue.equals(value)) { 1004 final String newValue = fixedValue; 1005 // misspelled preset value 1006 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE) 1007 .message(tr("Unknown property value"), 1008 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue) 1009 .primitives(p) 1010 .build()); 1011 withErrors.put(p, "WPV"); 1012 } else if (includeOtherSeverity) { 1013 // unknown preset value 1014 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 1015 .message(tr("Presets do not contain property value"), 1016 marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key) 1017 .primitives(p) 1018 .build()); 1019 withErrors.put(p, "UPV"); 1020 } 1021 } 1022 1023 // see #19180 1024 private void filterDeprecatedTags(OsmPrimitive p, String key, List<String> fixVals) { 1025 if (fixVals.isEmpty() || deprecatedChecker == null) 1026 return; 1027 1028 int unchangedDeprecated = countDeprecated(p); 1029 1030 // see #19895: create deep clone. This complex method works even with locked files 1031 MergeSourceBuildingVisitor builder = new MergeSourceBuildingVisitor(p.getDataSet()); 1032 p.accept(builder); 1033 DataSet clonedDs = builder.build(); 1034 OsmPrimitive clone = clonedDs.getPrimitiveById(p.getPrimitiveId()); 1035 1036 Iterator<String> iter = fixVals.iterator(); 1037 while (iter.hasNext()) { 1038 clone.put(key, iter.next()); 1039 if (countDeprecated(clone) > unchangedDeprecated) 1040 iter.remove(); 1041 } 1042 } 1043 1044 private int countDeprecated(OsmPrimitive p) { 1045 if (deprecatedChecker == null) 1046 return 0; 1047 deprecatedChecker.getErrors().clear(); 1048 deprecatedChecker.visit(Collections.singleton(p), url -> url.endsWith("deprecated.mapcss")); 1049 return deprecatedChecker.getErrors().size(); 1050 } 1051 1052 private static boolean isNum(String harmonizedValue) { 1053 try { 1054 Double.parseDouble(harmonizedValue); 1055 return true; 1056 } catch (NumberFormatException e) { 1057 return false; 1058 } 1059 } 1060 1061 private static boolean isFixme(String key, String value) { 1062 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo") 1063 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete"); 1064 } 1065 1066 private static String harmonizeKey(String key) { 1067 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,"); 1068 } 1069 1070 private static String harmonizeValue(String value) { 1071 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,"); 1072 } 1073 1074 @Override 1075 public void startTest(ProgressMonitor monitor) { 1076 super.startTest(monitor); 1077 includeOtherSeverity = includeOtherSeverityChecks(); 1078 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true); 1079 if (isBeforeUpload) { 1080 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true); 1081 } 1082 1083 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true); 1084 if (isBeforeUpload) { 1085 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true); 1086 } 1087 1088 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true); 1089 if (isBeforeUpload) { 1090 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true); 1091 } 1092 1093 checkFixmes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_FIXMES, true); 1094 if (isBeforeUpload) { 1095 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true); 1096 } 1097 1098 checkPresetsTypes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true); 1099 if (isBeforeUpload) { 1100 checkPresetsTypes = checkPresetsTypes && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true); 1101 } 1102 deprecatedChecker = OsmValidator.getTest(MapCSSTagChecker.class); 1103 ignoreForOuterMPSameTagCheck.addAll(Config.getPref().getList(PREF_KEYS_IGNORE_OUTER_MP_SAME_TAG, Collections.emptyList())); 1104 } 1105 1106 @Override 1107 public void endTest() { 1108 deprecatedChecker = null; 1109 super.endTest(); 1110 } 1111 1112 @Override 1113 public void visit(Collection<OsmPrimitive> selection) { 1114 if (checkKeys || checkValues || checkComplex || checkFixmes || checkPresetsTypes) { 1115 super.visit(selection); 1116 } 1117 } 1118 1119 @Override 1120 public void addGui(JPanel testPanel) { 1121 GBC a = GBC.eol(); 1122 a.anchor = GridBagConstraints.LINE_END; 1123 1124 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0)); 1125 1126 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true)); 1127 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words.")); 1128 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0)); 1129 1130 prefCheckKeysBeforeUpload = new JCheckBox(); 1131 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true)); 1132 testPanel.add(prefCheckKeysBeforeUpload, a); 1133 1134 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true)); 1135 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules.")); 1136 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0)); 1137 1138 prefCheckComplexBeforeUpload = new JCheckBox(); 1139 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true)); 1140 testPanel.add(prefCheckComplexBeforeUpload, a); 1141 1142 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES); 1143 sourcesList = new EditableList(tr("TagChecker source")); 1144 sourcesList.setItems(sources); 1145 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0)); 1146 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0)); 1147 1148 ActionListener disableCheckActionListener = e -> handlePrefEnable(); 1149 prefCheckKeys.addActionListener(disableCheckActionListener); 1150 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener); 1151 prefCheckComplex.addActionListener(disableCheckActionListener); 1152 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener); 1153 1154 handlePrefEnable(); 1155 1156 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true)); 1157 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets.")); 1158 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0)); 1159 1160 prefCheckValuesBeforeUpload = new JCheckBox(); 1161 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true)); 1162 testPanel.add(prefCheckValuesBeforeUpload, a); 1163 1164 prefCheckFixmes = new JCheckBox(tr("Check for fixme."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true)); 1165 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with fixme in any property value.")); 1166 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0)); 1167 1168 prefCheckFixmesBeforeUpload = new JCheckBox(); 1169 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true)); 1170 testPanel.add(prefCheckFixmesBeforeUpload, a); 1171 1172 prefCheckPresetsTypes = new JCheckBox(tr("Check for presets types."), Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true)); 1173 prefCheckPresetsTypes.setToolTipText(tr("Validate that objects types are valid checking against presets.")); 1174 testPanel.add(prefCheckPresetsTypes, GBC.std().insets(20, 0, 0, 0)); 1175 1176 prefCheckPresetsTypesBeforeUpload = new JCheckBox(); 1177 prefCheckPresetsTypesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true)); 1178 testPanel.add(prefCheckPresetsTypesBeforeUpload, a); 1179 } 1180 1181 /** 1182 * Enables/disables the source list field 1183 */ 1184 public void handlePrefEnable() { 1185 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected() 1186 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 1187 sourcesList.setEnabled(selected); 1188 } 1189 1190 @Override 1191 public boolean ok() { 1192 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected(); 1193 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected() 1194 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 1195 1196 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected()); 1197 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected()); 1198 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected()); 1199 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected()); 1200 Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES, prefCheckPresetsTypes.isSelected()); 1201 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected()); 1202 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected()); 1203 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected()); 1204 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected()); 1205 Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, prefCheckPresetsTypesBeforeUpload.isSelected()); 1206 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems()); 1207 } 1208 1209 @Override 1210 public Command fixError(TestError testError) { 1211 List<Command> commands = new ArrayList<>(50); 1212 1213 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives(); 1214 for (OsmPrimitive p : primitives) { 1215 Map<String, String> tags = p.getKeys(); 1216 if (tags.isEmpty()) { 1217 continue; 1218 } 1219 1220 for (Entry<String, String> prop: tags.entrySet()) { 1221 String key = prop.getKey(); 1222 String value = prop.getValue(); 1223 if (Utils.isBlank(value)) { 1224 commands.add(new ChangePropertyCommand(p, key, null)); 1225 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) { 1226 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value))); 1227 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) { 1228 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key))); 1229 } else { 1230 String evalue = Entities.unescape(value); 1231 if (!evalue.equals(value)) { 1232 commands.add(new ChangePropertyCommand(p, key, evalue)); 1233 } 1234 } 1235 } 1236 } 1237 1238 if (commands.isEmpty()) 1239 return null; 1240 if (commands.size() == 1) 1241 return commands.get(0); 1242 1243 return new SequenceCommand(tr("Fix tags"), commands); 1244 } 1245 1246 @Override 1247 public boolean isFixable(TestError testError) { 1248 if (testError.getTester() instanceof TagChecker) { 1249 int code = testError.getCode(); 1250 return code == EMPTY_VALUES || code == INVALID_SPACE || 1251 code == INVALID_KEY_SPACE || code == INVALID_HTML || 1252 code == MULTIPLE_SPACES; 1253 } 1254 1255 return false; 1256 } 1257 1258 @Override 1259 public void taggingPresetsModified() { 1260 try { 1261 initializeData(); 1262 initializePresets(); 1263 analysePresets(); 1264 } catch (IOException e) { 1265 Logging.error(e); 1266 } 1267 } 1268}