001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006import static org.openstreetmap.josm.tools.I18n.trn;
007
008import java.awt.GridBagConstraints;
009import java.awt.event.ActionListener;
010import java.io.BufferedReader;
011import java.io.IOException;
012import java.lang.Character.UnicodeBlock;
013import java.util.ArrayList;
014import java.util.Arrays;
015import java.util.Collection;
016import java.util.Collections;
017import java.util.EnumSet;
018import java.util.HashMap;
019import java.util.HashSet;
020import java.util.Iterator;
021import java.util.LinkedHashMap;
022import java.util.LinkedHashSet;
023import java.util.List;
024import java.util.Locale;
025import java.util.Map;
026import java.util.Map.Entry;
027import java.util.OptionalInt;
028import java.util.Set;
029import java.util.regex.Pattern;
030import java.util.stream.Collectors;
031
032import javax.swing.JCheckBox;
033import javax.swing.JLabel;
034import javax.swing.JPanel;
035
036import org.openstreetmap.josm.command.ChangePropertyCommand;
037import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
038import org.openstreetmap.josm.command.Command;
039import org.openstreetmap.josm.command.SequenceCommand;
040import org.openstreetmap.josm.data.osm.AbstractPrimitive;
041import org.openstreetmap.josm.data.osm.DataSet;
042import org.openstreetmap.josm.data.osm.OsmPrimitive;
043import org.openstreetmap.josm.data.osm.OsmUtils;
044import org.openstreetmap.josm.data.osm.Relation;
045import org.openstreetmap.josm.data.osm.RelationMember;
046import org.openstreetmap.josm.data.osm.Tag;
047import org.openstreetmap.josm.data.osm.TagMap;
048import org.openstreetmap.josm.data.osm.Tagged;
049import org.openstreetmap.josm.data.osm.Way;
050import org.openstreetmap.josm.data.osm.visitor.MergeSourceBuildingVisitor;
051import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
052import org.openstreetmap.josm.data.validation.OsmValidator;
053import org.openstreetmap.josm.data.validation.Severity;
054import org.openstreetmap.josm.data.validation.Test.TagTest;
055import org.openstreetmap.josm.data.validation.TestError;
056import org.openstreetmap.josm.data.validation.util.Entities;
057import org.openstreetmap.josm.gui.progress.ProgressMonitor;
058import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
059import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
060import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetListener;
061import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetType;
062import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
063import org.openstreetmap.josm.gui.tagging.presets.items.Check;
064import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
065import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
066import org.openstreetmap.josm.gui.widgets.EditableList;
067import org.openstreetmap.josm.io.CachedFile;
068import org.openstreetmap.josm.spi.preferences.Config;
069import org.openstreetmap.josm.tools.GBC;
070import org.openstreetmap.josm.tools.Logging;
071import org.openstreetmap.josm.tools.MultiMap;
072import org.openstreetmap.josm.tools.Utils;
073
074/**
075 * Check for misspelled or wrong tags
076 *
077 * @author frsantos
078 * @since 3669
079 */
080public class TagChecker extends TagTest implements TaggingPresetListener {
081
082    /** The config file of ignored tags */
083    public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
084    /** The config file of dictionary words */
085    public static final String SPELL_FILE = "resource://data/validator/words.cfg";
086
087    /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
088    private static final Map<String, String> harmonizedKeys = new HashMap<>();
089    /** The spell check preset values which are not stored in TaggingPresets */
090    private static volatile HashSet<String> additionalPresetsValueData;
091    /** often used tags which are not in presets */
092    private static final MultiMap<String, String> oftenUsedTags = new MultiMap<>();
093    private static final Map<TaggingPreset, List<TaggingPresetItem>> presetIndex = new LinkedHashMap<>();
094
095    private static final Pattern UNWANTED_NON_PRINTING_CONTROL_CHARACTERS = Pattern.compile(
096            "[\\x00-\\x09\\x0B\\x0C\\x0E-\\x1F\\x7F\\u200e-\\u200f\\u202a-\\u202e]");
097
098    /** The TagChecker data */
099    private static final List<String> ignoreDataStartsWith = new ArrayList<>();
100    private static final Set<String> ignoreDataEquals = new HashSet<>();
101    private static final List<String> ignoreDataEndsWith = new ArrayList<>();
102    private static final List<Tag> ignoreDataTag = new ArrayList<>();
103    /** tag keys that have only numerical values in the presets */
104    private static final Set<String> ignoreForLevenshtein = new HashSet<>();
105
106    /** tag keys that are allowed to be the same on a multipolygon and an outer way */
107    private static final Set<String> ignoreForOuterMPSameTagCheck = new HashSet<>();
108
109    /** The preferences prefix */
110    protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
111
112    MapCSSTagChecker deprecatedChecker;
113
114    /**
115     * The preference key to check values
116     */
117    public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
118    /**
119     * The preference key to check keys
120     */
121    public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
122    /**
123     * The preference key to enable complex checks
124     */
125    public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
126    /**
127     * The preference key to search for fixme tags
128     */
129    public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
130    /**
131     * The preference key to check presets
132     */
133    public static final String PREF_CHECK_PRESETS_TYPES = PREFIX + ".checkPresetsTypes";
134
135    /**
136     * The preference key for source files
137     * @see #DEFAULT_SOURCES
138     */
139    public static final String PREF_SOURCES = PREFIX + ".source";
140
141    private static final String BEFORE_UPLOAD = "BeforeUpload";
142    /**
143     * The preference key to check keys - used before upload
144     */
145    public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + BEFORE_UPLOAD;
146    /**
147     * The preference key to check values - used before upload
148     */
149    public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + BEFORE_UPLOAD;
150    /**
151     * The preference key to run complex tests - used before upload
152     */
153    public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + BEFORE_UPLOAD;
154    /**
155     * The preference key to search for fixmes - used before upload
156     */
157    public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + BEFORE_UPLOAD;
158    /**
159     * The preference key to search for presets - used before upload
160     */
161    public static final String PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD = PREF_CHECK_PRESETS_TYPES + BEFORE_UPLOAD;
162
163    /**
164     * The preference key for the list of tag keys that are allowed to be the same on a multipolygon and an outer way
165     */
166    public static final String PREF_KEYS_IGNORE_OUTER_MP_SAME_TAG = PREFIX + ".ignore-keys-outer-mp-same-tag";
167
168    private static final int MAX_LEVENSHTEIN_DISTANCE = 2;
169
170    protected boolean includeOtherSeverity;
171
172    protected boolean checkKeys;
173    protected boolean checkValues;
174    /** Was used for special configuration file, might be used to disable value spell checker. */
175    protected boolean checkComplex;
176    protected boolean checkFixmes;
177    protected boolean checkPresetsTypes;
178
179    protected JCheckBox prefCheckKeys;
180    protected JCheckBox prefCheckValues;
181    protected JCheckBox prefCheckComplex;
182    protected JCheckBox prefCheckFixmes;
183    protected JCheckBox prefCheckPresetsTypes;
184
185    protected JCheckBox prefCheckKeysBeforeUpload;
186    protected JCheckBox prefCheckValuesBeforeUpload;
187    protected JCheckBox prefCheckComplexBeforeUpload;
188    protected JCheckBox prefCheckFixmesBeforeUpload;
189    protected JCheckBox prefCheckPresetsTypesBeforeUpload;
190
191    // CHECKSTYLE.OFF: SingleSpaceSeparator
192    protected static final int EMPTY_VALUES                     = 1200;
193    protected static final int INVALID_KEY                      = 1201;
194    protected static final int INVALID_VALUE                    = 1202;
195    protected static final int FIXME                            = 1203;
196    protected static final int INVALID_SPACE                    = 1204;
197    protected static final int INVALID_KEY_SPACE                = 1205;
198    protected static final int INVALID_HTML                     = 1206; /* 1207 was PAINT */
199    protected static final int LONG_VALUE                       = 1208;
200    protected static final int LONG_KEY                         = 1209;
201    protected static final int LOW_CHAR_VALUE                   = 1210;
202    protected static final int LOW_CHAR_KEY                     = 1211;
203    protected static final int MISSPELLED_VALUE                 = 1212;
204    protected static final int MISSPELLED_KEY                   = 1213;
205    protected static final int MULTIPLE_SPACES                  = 1214;
206    protected static final int MISSPELLED_VALUE_NO_FIX          = 1215;
207    protected static final int UNUSUAL_UNICODE_CHAR_VALUE       = 1216;
208    protected static final int INVALID_PRESETS_TYPE             = 1217;
209    protected static final int MULTIPOLYGON_NO_AREA             = 1218;
210    protected static final int MULTIPOLYGON_INCOMPLETE          = 1219;
211    protected static final int MULTIPOLYGON_MAYBE_NO_AREA       = 1220;
212    protected static final int MULTIPOLYGON_SAME_TAG_ON_OUTER   = 1221;
213    // CHECKSTYLE.ON: SingleSpaceSeparator
214
215    protected EditableList sourcesList;
216
217    private static final List<String> DEFAULT_SOURCES = Arrays.asList(IGNORE_FILE, SPELL_FILE);
218
219    /**
220     * Constructor
221     */
222    public TagChecker() {
223        super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
224    }
225
226    @Override
227    public void initialize() throws IOException {
228        TaggingPresets.addListener(this);
229        initializeData();
230        initializePresets();
231        analysePresets();
232    }
233
234    /**
235     * Add presets that contain only numerical values to the ignore list
236     */
237    private static void analysePresets() {
238        for (String key : TaggingPresets.getPresetKeys()) {
239            if (isKeyIgnored(key))
240                continue;
241            Set<String> values = TaggingPresets.getPresetValues(key);
242            boolean allNumerical = !Utils.isEmpty(values)
243                    && values.stream().allMatch(TagChecker::isNum);
244            if (allNumerical) {
245                ignoreForLevenshtein.add(key);
246            }
247        }
248    }
249
250    /**
251     * Reads the spell-check file into a HashMap.
252     * The data file is a list of words, beginning with +/-. If it starts with +,
253     * the word is valid, but if it starts with -, the word should be replaced
254     * by the nearest + word before this.
255     *
256     * @throws IOException if any I/O error occurs
257     */
258    private static void initializeData() throws IOException {
259        ignoreDataStartsWith.clear();
260        ignoreDataEquals.clear();
261        ignoreDataEndsWith.clear();
262        ignoreDataTag.clear();
263        harmonizedKeys.clear();
264        ignoreForLevenshtein.clear();
265        oftenUsedTags.clear();
266        presetIndex.clear();
267        ignoreForOuterMPSameTagCheck.clear();
268
269        StringBuilder errorSources = new StringBuilder();
270        for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
271            try (
272                CachedFile cf = new CachedFile(source);
273                BufferedReader reader = cf.getContentReader()
274            ) {
275                String okValue = null;
276                boolean tagcheckerfile = false;
277                boolean ignorefile = false;
278                boolean isFirstLine = true;
279                String line;
280                while ((line = reader.readLine()) != null) {
281                    if (line.isEmpty()) {
282                        // ignore
283                    } else if (line.startsWith("#")) {
284                        if (line.startsWith("# JOSM TagChecker")) {
285                            tagcheckerfile = true;
286                            Logging.error(tr("Ignoring {0}. Support was dropped", source));
287                        } else
288                        if (line.startsWith("# JOSM IgnoreTags")) {
289                            ignorefile = true;
290                            if (!DEFAULT_SOURCES.contains(source)) {
291                                Logging.info(tr("Adding {0} to ignore tags", source));
292                            }
293                        }
294                    } else if (ignorefile) {
295                        parseIgnoreFileLine(source, line);
296                    } else if (tagcheckerfile) {
297                        // ignore
298                    } else if (line.charAt(0) == '+') {
299                        okValue = line.substring(1);
300                    } else if (line.charAt(0) == '-' && okValue != null) {
301                        String hk = harmonizeKey(line.substring(1));
302                        if (!okValue.equals(hk) && harmonizedKeys.put(hk, okValue) != null && Logging.isDebugEnabled()) {
303                            Logging.debug("Line was ignored: " + line);
304                        }
305                    } else {
306                        Logging.error(tr("Invalid spellcheck line: {0}", line));
307                    }
308                    if (isFirstLine) {
309                        isFirstLine = false;
310                        if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
311                            Logging.info(tr("Adding {0} to spellchecker", source));
312                        }
313                    }
314                }
315            } catch (IOException e) {
316                Logging.error(e);
317                errorSources.append(source).append('\n');
318            }
319        }
320
321        if (errorSources.length() > 0)
322            throw new IOException(trn(
323                    "Could not access data file:\n{0}",
324                    "Could not access data files:\n{0}", errorSources.length(), errorSources));
325    }
326
327    /**
328     * Parse a line found in a configuration file
329     * @param source name of configuration file
330     * @param line the line to parse
331     */
332    private static void parseIgnoreFileLine(String source, String line) {
333        line = line.trim();
334        if (line.length() < 4) {
335            return;
336        }
337        try {
338            String key = line.substring(0, 2);
339            line = line.substring(2);
340
341            switch (key) {
342            case "S:":
343                ignoreDataStartsWith.add(line);
344                break;
345            case "E:":
346                ignoreDataEquals.add(line);
347                addToKeyDictionary(line);
348                break;
349            case "F:":
350                ignoreDataEndsWith.add(line);
351                break;
352            case "K:":
353                Tag tag = Tag.ofString(line);
354                ignoreDataTag.add(tag);
355                oftenUsedTags.put(tag.getKey(), tag.getValue());
356                addToKeyDictionary(tag.getKey());
357                break;
358            default:
359                if (!key.startsWith(";")) {
360                    Logging.warn("Unsupported TagChecker key: " + key);
361                }
362            }
363        } catch (IllegalArgumentException e) {
364            Logging.error("Invalid line in {0} : {1}", source, e.getMessage());
365            Logging.trace(e);
366        }
367    }
368
369    private static void addToKeyDictionary(String key) {
370        if (key != null) {
371            String hk = harmonizeKey(key);
372            if (!key.equals(hk)) {
373                harmonizedKeys.put(hk, key);
374            }
375        }
376    }
377
378    /**
379     * Reads the presets data.
380     *
381     */
382    public static void initializePresets() {
383
384        if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
385            return;
386
387        Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
388        if (!presets.isEmpty()) {
389            initAdditionalPresetsValueData();
390            for (TaggingPreset p : presets) {
391                List<TaggingPresetItem> minData = new ArrayList<>();
392                for (TaggingPresetItem i : p.data) {
393                    if (i instanceof KeyedItem) {
394                        if (!"none".equals(((KeyedItem) i).match))
395                            minData.add(i);
396                        addPresetValue((KeyedItem) i);
397                    } else if (i instanceof CheckGroup) {
398                        for (Check c : ((CheckGroup) i).checks) {
399                            addPresetValue(c);
400                        }
401                    }
402                }
403                if (!minData.isEmpty()) {
404                    presetIndex .put(p, minData);
405                }
406            }
407        }
408    }
409
410    private static void initAdditionalPresetsValueData() {
411        additionalPresetsValueData = new HashSet<>();
412        additionalPresetsValueData.addAll(AbstractPrimitive.getUninterestingKeys());
413        additionalPresetsValueData.addAll(Config.getPref().getList(
414                ValidatorPrefHelper.PREFIX + ".knownkeys",
415                Arrays.asList("is_in", "int_ref", "fixme", "population")));
416    }
417
418    private static void addPresetValue(KeyedItem ky) {
419        if (ky.key != null && ky.getValues() != null) {
420            addToKeyDictionary(ky.key);
421        }
422    }
423
424    /**
425     * Checks given string (key or value) if it contains unwanted non-printing control characters (either ASCII or Unicode bidi characters)
426     * @param s string to check
427     * @return {@code true} if {@code s} contains non-printing control characters
428     */
429    static boolean containsUnwantedNonPrintingControlCharacter(String s) {
430        return !Utils.isEmpty(s) && (
431                isJoiningChar(s.charAt(0)) ||
432                isJoiningChar(s.charAt(s.length() - 1)) ||
433                s.chars().anyMatch(c -> (isAsciiControlChar(c) && !isNewLineChar(c)) || isBidiControlChar(c))
434                );
435    }
436
437    private static boolean isAsciiControlChar(int c) {
438        return c < 0x20 || c == 0x7F;
439    }
440
441    private static boolean isNewLineChar(int c) {
442        return c == 0x0a || c == 0x0d;
443    }
444
445    private static boolean isJoiningChar(int c) {
446        return c == 0x200c || c == 0x200d; // ZWNJ, ZWJ
447    }
448
449    private static boolean isBidiControlChar(int c) {
450        /* check for range 0x200e to 0x200f (LRM, RLM) or
451                           0x202a to 0x202e (LRE, RLE, PDF, LRO, RLO) */
452        return (c >= 0x200e && c <= 0x200f) || (c >= 0x202a && c <= 0x202e);
453    }
454
455    static String removeUnwantedNonPrintingControlCharacters(String s) {
456        // Remove all unwanted characters
457        String result = UNWANTED_NON_PRINTING_CONTROL_CHARACTERS.matcher(s).replaceAll("");
458        // Remove joining characters located at the beginning of the string
459        while (!result.isEmpty() && isJoiningChar(result.charAt(0))) {
460            result = result.substring(1);
461        }
462        // Remove joining characters located at the end of the string
463        while (!result.isEmpty() && isJoiningChar(result.charAt(result.length() - 1))) {
464            result = result.substring(0, result.length() - 1);
465        }
466        return result;
467    }
468
469    static boolean containsUnusualUnicodeCharacter(String key, String value) {
470        return getUnusualUnicodeCharacter(key, value).isPresent();
471    }
472
473    static OptionalInt getUnusualUnicodeCharacter(String key, String value) {
474        return value == null
475                ? OptionalInt.empty()
476                : value.chars().filter(c -> isUnusualUnicodeBlock(key, c)).findFirst();
477    }
478
479    /**
480     * Detects highly suspicious Unicode characters that have been seen in OSM database.
481     * @param key tag key
482     * @param c current character code point
483     * @return {@code true} if the current unicode block is very unusual for the given key
484     */
485    private static boolean isUnusualUnicodeBlock(String key, int c) {
486        UnicodeBlock b = UnicodeBlock.of(c);
487        return isUnusualPhoneticUse(key, b, c) || isUnusualBmpUse(b) || isUnusualSmpUse(b);
488    }
489
490    private static boolean isAllowedPhoneticCharacter(String key, int c) {
491        // CHECKSTYLE.OFF: BooleanExpressionComplexity
492        return c == 0x0259 || c == 0x018F // U+0259 is paired with the capital letter U+018F in Azeri, see #18740
493            || c == 0x0254 || c == 0x0186 // U+0254 is paired with the capital letter U+0186 in several African languages, see #18740
494            || c == 0x0257 || c == 0x018A // "ɗ/Ɗ" (U+0257/U+018A), see #19760
495            || c == 0x025B || c == 0x0190 // U+025B is paired with the capital letter U+0190 in several African languages, see #18740
496            || c == 0x0263 || c == 0x0194 // "ɣ/Ɣ" (U+0263/U+0194), see #18740
497            || c == 0x0268 || c == 0x0197 // "ɨ/Ɨ" (U+0268/U+0197), see #18740
498            || c == 0x0269 || c == 0x0196 // "ɩ/Ɩ" (U+0269/U+0196), see #20437
499            || c == 0x0272 || c == 0x019D // "ɲ/Ɲ" (U+0272/U+019D), see #18740
500            || c == 0x0273 || c == 0x019E // "ŋ/Ŋ" (U+0273/U+019E), see #18740
501            || c == 0x0142 || c == 0x0294 // see #20754
502            || (key.endsWith("ref") && 0x1D2C <= c && c <= 0x1D42); // allow uppercase superscript latin characters in *ref tags
503    }
504
505    private static boolean isUnusualPhoneticUse(String key, UnicodeBlock b, int c) {
506        return !isAllowedPhoneticCharacter(key, c)
507            && (b == UnicodeBlock.IPA_EXTENSIONS                        // U+0250..U+02AF
508             || b == UnicodeBlock.PHONETIC_EXTENSIONS                   // U+1D00..U+1D7F
509             || b == UnicodeBlock.PHONETIC_EXTENSIONS_SUPPLEMENT)       // U+1D80..U+1DBF
510                && !key.endsWith(":pronunciation");
511    }
512
513    private static boolean isUnusualBmpUse(UnicodeBlock b) {
514        return b == UnicodeBlock.COMBINING_MARKS_FOR_SYMBOLS            // U+20D0..U+20FF
515            || b == UnicodeBlock.MATHEMATICAL_OPERATORS                 // U+2200..U+22FF
516            || b == UnicodeBlock.ENCLOSED_ALPHANUMERICS                 // U+2460..U+24FF
517            || b == UnicodeBlock.BOX_DRAWING                            // U+2500..U+257F
518            || b == UnicodeBlock.GEOMETRIC_SHAPES                       // U+25A0..U+25FF
519            || b == UnicodeBlock.DINGBATS                               // U+2700..U+27BF
520            || b == UnicodeBlock.MISCELLANEOUS_SYMBOLS_AND_ARROWS       // U+2B00..U+2BFF
521            || b == UnicodeBlock.GLAGOLITIC                             // U+2C00..U+2C5F
522            || b == UnicodeBlock.HANGUL_COMPATIBILITY_JAMO              // U+3130..U+318F
523            || b == UnicodeBlock.ENCLOSED_CJK_LETTERS_AND_MONTHS        // U+3200..U+32FF
524            || b == UnicodeBlock.LATIN_EXTENDED_D                       // U+A720..U+A7FF
525            || b == UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS           // U+F900..U+FAFF
526            || b == UnicodeBlock.ALPHABETIC_PRESENTATION_FORMS          // U+FB00..U+FB4F
527            || b == UnicodeBlock.VARIATION_SELECTORS                    // U+FE00..U+FE0F
528            || b == UnicodeBlock.SPECIALS;                              // U+FFF0..U+FFFF
529            // CHECKSTYLE.ON: BooleanExpressionComplexity
530    }
531
532    private static boolean isUnusualSmpUse(UnicodeBlock b) {
533        // UnicodeBlock.SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS is only defined in Java 9+
534        return b == UnicodeBlock.MUSICAL_SYMBOLS                        // U+1D100..U+1D1FF
535            || b == UnicodeBlock.ENCLOSED_ALPHANUMERIC_SUPPLEMENT       // U+1F100..U+1F1FF
536            || b == UnicodeBlock.EMOTICONS                              // U+1F600..U+1F64F
537            || b == UnicodeBlock.TRANSPORT_AND_MAP_SYMBOLS;             // U+1F680..U+1F6FF
538    }
539
540    /**
541     * Get set of preset values for the given key.
542     * @param key the key
543     * @return null if key is not in presets or in additionalPresetsValueData,
544     *  else a set which might be empty.
545     */
546    private static Set<String> getPresetValues(String key) {
547        if (TaggingPresets.isKeyInPresets(key)) {
548            return TaggingPresets.getPresetValues(key);
549        }
550        if (additionalPresetsValueData.contains(key))
551            return Collections.emptySet();
552        // null means key is not known
553        return null;
554    }
555
556    /**
557     * Determines if the given key is in internal presets.
558     * @param key key
559     * @return {@code true} if the given key is in internal presets
560     * @since 9023
561     * @deprecated Use {@link TaggingPresets#isKeyInPresets(String)} instead
562     */
563    @Deprecated
564    public static boolean isKeyInPresets(String key) {
565        return TaggingPresets.isKeyInPresets(key);
566    }
567
568    /**
569     * Determines if the given tag is in internal presets.
570     * @param key key
571     * @param value value
572     * @return {@code true} if the given tag is in internal presets
573     * @since 9023
574     */
575    public static boolean isTagInPresets(String key, String value) {
576        final Set<String> values = getPresetValues(key);
577        return values != null && values.contains(value);
578    }
579
580    /**
581     * Returns the list of ignored tags.
582     * @return the list of ignored tags
583     * @since 9023
584     */
585    public static List<Tag> getIgnoredTags() {
586        return new ArrayList<>(ignoreDataTag);
587    }
588
589    /**
590     * Determines if the given tag key is ignored for checks "key/tag not in presets".
591     * @param key key
592     * @return true if the given key is ignored
593     */
594    private static boolean isKeyIgnored(String key) {
595        return ignoreDataEquals.contains(key)
596                || ignoreDataStartsWith.stream().anyMatch(key::startsWith)
597                || ignoreDataEndsWith.stream().anyMatch(key::endsWith);
598    }
599
600    /**
601     * Determines if the given tag is ignored for checks "key/tag not in presets".
602     * @param key key
603     * @param value value
604     * @return {@code true} if the given tag is ignored
605     * @since 9023
606     */
607    public static boolean isTagIgnored(String key, String value) {
608        if (isKeyIgnored(key))
609            return true;
610        final Set<String> values = getPresetValues(key);
611        if (values != null && values.isEmpty())
612            return true;
613        if (!isTagInPresets(key, value)) {
614            return ignoreDataTag.stream()
615                    .anyMatch(a -> key.equals(a.getKey()) && value.equals(a.getValue()));
616        }
617        return false;
618    }
619
620    /**
621     * Checks the primitive tags
622     * @param p The primitive to check
623     */
624    @Override
625    public void check(OsmPrimitive p) {
626        if (!p.isTagged())
627            return;
628
629        // Just a collection to know if a primitive has been already marked with error
630        MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
631
632        for (Entry<String, String> prop : p.getKeys().entrySet()) {
633            String s = marktr("Tag ''{0}'' invalid.");
634            String key = prop.getKey();
635            String value = prop.getValue();
636
637            if (checkKeys) {
638                checkSingleTagKeySimple(withErrors, p, s, key);
639            }
640            if (checkValues) {
641                checkSingleTagValueSimple(withErrors, p, s, key, value);
642                checkSingleTagComplex(withErrors, p, key, value);
643            }
644            if (checkFixmes && key != null && !Utils.isEmpty(value) && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
645                errors.add(TestError.builder(this, Severity.OTHER, FIXME)
646                        .message(tr("fixme"))
647                        .primitives(p)
648                        .build());
649                withErrors.put(p, "FIXME");
650            }
651        }
652
653        if (p instanceof Relation && p.hasTag("type", "multipolygon")) {
654        checkMultipolygonTags(p);
655        }
656
657        if (checkPresetsTypes) {
658            TagMap tags = p.getKeys();
659            TaggingPresetType presetType = TaggingPresetType.forPrimitive(p);
660            EnumSet<TaggingPresetType> presetTypes = EnumSet.of(presetType);
661
662            Collection<TaggingPreset> matchingPresets = presetIndex.entrySet().stream()
663                    .filter(e -> TaggingPresetItem.matches(e.getValue(), tags))
664                    .map(Entry::getKey)
665                    .collect(Collectors.toCollection(LinkedHashSet::new));
666            Collection<TaggingPreset> matchingPresetsOK = matchingPresets.stream().filter(
667                    tp -> tp.typeMatches(presetTypes)).collect(Collectors.toList());
668            Collection<TaggingPreset> matchingPresetsKO = matchingPresets.stream().filter(
669                    tp -> !tp.typeMatches(presetTypes)).collect(Collectors.toList());
670
671            for (TaggingPreset tp : matchingPresetsKO) {
672                // Potential error, unless matching tags are all known by a supported preset
673                Map<String, String> matchingTags = tp.data.stream()
674                    .filter(i -> Boolean.TRUE.equals(i.matches(tags)))
675                    .filter(i -> i instanceof KeyedItem).map(i -> ((KeyedItem) i).key)
676                    .collect(Collectors.toMap(k -> k, tags::get));
677                if (matchingPresetsOK.stream().noneMatch(
678                        tp2 -> matchingTags.entrySet().stream().allMatch(
679                                e -> tp2.data.stream().anyMatch(
680                                        i -> i instanceof KeyedItem && ((KeyedItem) i).key.equals(e.getKey()))))) {
681                    errors.add(TestError.builder(this, Severity.OTHER, INVALID_PRESETS_TYPE)
682                            .message(tr("Object type not in preset"),
683                                    marktr("Object type {0} is not supported by tagging preset: {1}"),
684                                    tr(presetType.getName()), tp.getLocaleName())
685                            .primitives(p)
686                            .build());
687                }
688            }
689        }
690    }
691
692    private static final Collection<String> NO_AREA_KEYS = Arrays.asList("name", "area", "ref", "access", "operator");
693
694    private void checkMultipolygonTags(OsmPrimitive p) {
695        if (p.isAnnotated() || p.keys()
696                .anyMatch(k -> k.matches("^(abandoned|construction|demolished|disused|planned|razed|removed|was).*")))
697            return;
698
699        checkOuterWaysOfRelation((Relation) p);
700
701        if (hasAcceptedPrimaryTagForMultipolygon(p))
702            return;
703        TestError.Builder builder = null;
704        if (p.hasKey("surface")) {
705            // accept often used tag surface=* as area tag
706            builder = TestError.builder(this, Severity.OTHER, MULTIPOLYGON_INCOMPLETE)
707                    .message(tr("Multipolygon tags"), marktr("only {0} tag"), "surface");
708        } else {
709            Map<String, String> filteredTags = p.getInterestingTags();
710            filteredTags.remove("type");
711            NO_AREA_KEYS.forEach(filteredTags::remove);
712            filteredTags.keySet().removeIf(key -> !key.matches("[a-z0-9:_]+"));
713
714            if (filteredTags.isEmpty()) {
715                builder = TestError.builder(this, Severity.ERROR, MULTIPOLYGON_NO_AREA)
716                        .message(tr("Multipolygon tags"), marktr("tag describing the area is missing"), new Object());
717
718            }
719        }
720        if (builder == null) {
721            // multipolygon has either no area tag or a rarely used one
722            builder = TestError.builder(this, Severity.WARNING, MULTIPOLYGON_MAYBE_NO_AREA)
723                    .message(tr("Multipolygon tags"), marktr("tag describing the area might be missing"), new Object());
724        }
725        errors.add(builder.primitives(p).build());
726    }
727
728    /**
729     * Check if an outer way of the relation has the same tag as the relation.
730     * @param rel the relation
731     */
732    private void checkOuterWaysOfRelation(Relation rel) {
733        for (Entry<String, String> tag : rel.getInterestingTags().entrySet()) {
734            if (ignoreForOuterMPSameTagCheck.contains(tag.getKey()))
735                continue;
736
737            Set<Way> sameOuters = rel.getMembers().stream()
738                    .filter(rm -> rm.isWay() && rm.getWay().isArea() && "outer".equals(rm.getRole())
739                            && tag.getValue().equals(rm.getWay().get(tag.getKey())))
740                    .map(RelationMember::getWay).collect(Collectors.toSet());
741            if (!sameOuters.isEmpty()) {
742                List<OsmPrimitive> primitives = new ArrayList<>(sameOuters.size() + 1);
743                primitives.add(rel);
744                primitives.addAll(sameOuters);
745                Way w = new Way();
746                w.put(tag.getKey(), tag.getValue());
747                if (hasAcceptedPrimaryTagForMultipolygon(w)) {
748                    errors.add(TestError.builder(this, Severity.WARNING, MULTIPOLYGON_SAME_TAG_ON_OUTER)
749                            .message(tr("Multipolygon outer way repeats major tag of relation"),
750                                    marktr("Same tag:''{0}''=''{1}''"), tag.getKey(), tag.getValue())
751                            .primitives(primitives)
752                            .build());
753                } else {
754                    errors.add(TestError.builder(this, Severity.OTHER, MULTIPOLYGON_SAME_TAG_ON_OUTER)
755                            .message(tr("Multipolygon outer way repeats tag of relation"),
756                                    marktr("Same tag:''{0}''=''{1}''"), tag.getKey(), tag.getValue())
757                            .primitives(primitives)
758                            .build());
759                }
760            }
761        }
762    }
763
764    /**
765     * Check if a multipolygon has a main tag that describes the type of area. Accepts also some deprecated tags and typos.
766     * @param p the multipolygon
767     * @return true if the multipolygon has a main tag that (likely) describes the type of area.
768     */
769    private static boolean hasAcceptedPrimaryTagForMultipolygon(OsmPrimitive p) {
770        if (p.hasKey("landuse", "amenity", "building", "building:part", "area:highway", "shop", "place", "boundary",
771                "landform", "piste:type", "sport", "golf", "landcover", "aeroway", "office", "healthcare", "craft", "room")
772                || p.hasTagDifferent("natural", "tree", "peek", "saddle", "tree_row")
773                || p.hasTagDifferent("man_made", "survey_point", "mast", "flagpole", "manhole", "watertap")
774                || p.hasTagDifferent("highway", "crossing", "bus_stop", "turning_circle", "street_lamp",
775                        "traffic_signals", "stop", "milestone", "mini_roundabout", "motorway_junction", "passing_place",
776                        "speed_camera", "traffic_mirror", "trailhead", "turning_circle", "turning_loop", "toll_gantry")
777                || p.hasTagDifferent("tourism", "attraction", "artwork")
778                || p.hasTagDifferent("leisure", "picnic_table", "slipway", "firepit")
779                || p.hasTagDifferent("historic", "wayside_cross", "milestone"))
780            return true;
781        if (p.hasTag("barrier", "hedge", "retaining_wall")
782                || p.hasTag("public_transport", "platform", "station")
783                || p.hasTag("railway", "platform")
784                || p.hasTag("waterway", "riverbank", "dam", "rapids", "dock", "boatyard", "fuel")
785                || p.hasTag("indoor", "corridor", "room", "area")
786                || p.hasTag("power", "substation", "generator", "plant", "switchgear", "converter", "sub_station")
787                || p.hasTag("seamark:type", "harbour", "fairway", "anchorage", "landmark", "berth", "harbour_basin",
788                        "separation_zone")
789                || (p.get("seamark:type") != null && p.get("seamark:type").matches(".*\\_(area|zone)$")))
790            return true;
791        return p.hasTag("harbour", OsmUtils.TRUE_VALUE)
792                || p.hasTag("flood_prone", OsmUtils.TRUE_VALUE)
793                || p.hasTag("bridge", OsmUtils.TRUE_VALUE)
794                || p.hasTag("ruins", OsmUtils.TRUE_VALUE)
795                || p.hasTag("junction", OsmUtils.TRUE_VALUE);
796    }
797
798    private void checkSingleTagValueSimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key, String value) {
799        if (!checkValues || value == null)
800            return;
801        if (containsUnwantedNonPrintingControlCharacter(value) && !withErrors.contains(p, "ICV")) {
802            errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
803                    .message(tr("Tag value contains non-printing (usually invisible) character"), s, key)
804                    .primitives(p)
805                    .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(value)))
806                    .build());
807            withErrors.put(p, "ICV");
808        }
809        final OptionalInt unusualUnicodeCharacter = getUnusualUnicodeCharacter(key, value);
810        if (unusualUnicodeCharacter.isPresent() && !withErrors.contains(p, "UUCV")) {
811            final String codepoint = String.format(Locale.ROOT, "U+%04X", unusualUnicodeCharacter.getAsInt());
812            errors.add(TestError.builder(this, Severity.WARNING, UNUSUAL_UNICODE_CHAR_VALUE)
813                    .message(tr("Tag value contains unusual Unicode character {0}", codepoint), s, key)
814                    .primitives(p)
815                    .build());
816            withErrors.put(p, "UUCV");
817        }
818        if ((value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
819            errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
820                    .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
821                    .primitives(p)
822                    .build());
823            withErrors.put(p, "LV");
824        }
825        if (value.trim().isEmpty() && !withErrors.contains(p, "EV")) {
826            errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
827                    .message(tr("Tags with empty values"), s, key)
828                    .primitives(p)
829                    .build());
830            withErrors.put(p, "EV");
831        }
832        final String errTypeSpace = "SPACE";
833        if ((value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, errTypeSpace)) {
834            errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
835                    .message(tr("Property values start or end with white space"), s, key)
836                    .primitives(p)
837                    .build());
838            withErrors.put(p, errTypeSpace);
839        }
840        if (value.contains("  ") && !withErrors.contains(p, errTypeSpace)) {
841            errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
842                    .message(tr("Property values contain multiple white spaces"), s, key)
843                    .primitives(p)
844                    .build());
845            withErrors.put(p, errTypeSpace);
846        }
847        if (includeOtherSeverity && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
848            errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
849                    .message(tr("Property values contain HTML entity"), s, key)
850                    .primitives(p)
851                    .build());
852            withErrors.put(p, "HTML");
853        }
854    }
855
856    private void checkSingleTagKeySimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key) {
857        if (!checkKeys || key == null)
858            return;
859        if (containsUnwantedNonPrintingControlCharacter(key) && !withErrors.contains(p, "ICK")) {
860            errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
861                    .message(tr("Tag key contains non-printing character"), s, key)
862                    .primitives(p)
863                    .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(key)))
864                    .build());
865            withErrors.put(p, "ICK");
866        }
867        if (key.length() > Tagged.MAX_TAG_LENGTH && !withErrors.contains(p, "LK")) {
868            errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
869                    .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
870                    .primitives(p)
871                    .build());
872            withErrors.put(p, "LK");
873        }
874        if (key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
875            errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
876                    .message(tr("Invalid white space in property key"), s, key)
877                    .primitives(p)
878                    .build());
879            withErrors.put(p, "IPK");
880        }
881    }
882
883    private void checkSingleTagComplex(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key, String value) {
884        if (!checkValues || key == null || Utils.isEmpty(value))
885            return;
886        if (additionalPresetsValueData != null && !isTagIgnored(key, value)) {
887            if (!TaggingPresets.isKeyInPresets(key)) {
888                spellCheckKey(withErrors, p, key);
889            } else if (!isTagInPresets(key, value)) {
890                if (oftenUsedTags.contains(key, value)) {
891                    // tag is quite often used but not in presets
892                    errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
893                            .message(tr("Presets do not contain property value"),
894                                    marktr("Value ''{0}'' for key ''{1}'' not in presets, but is known."), value, key)
895                            .primitives(p)
896                            .build());
897                    withErrors.put(p, "UPV");
898                } else {
899                    tryGuess(p, key, value, withErrors);
900                }
901            }
902        }
903    }
904
905    private void spellCheckKey(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key) {
906        String prettifiedKey = harmonizeKey(key);
907        String fixedKey;
908        if (ignoreDataEquals.contains(prettifiedKey)) {
909            fixedKey = prettifiedKey;
910        } else {
911            fixedKey = TaggingPresets.isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey);
912        }
913        if (fixedKey == null && ignoreDataTag.stream().anyMatch(a -> a.getKey().equals(prettifiedKey))) {
914            fixedKey = prettifiedKey;
915        }
916
917        if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
918            final String proposedKey = fixedKey;
919            // misspelled preset key
920            final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
921                    .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, proposedKey)
922                    .primitives(p);
923            if (p.hasKey(fixedKey)) {
924                errors.add(error.build());
925            } else {
926                errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, proposedKey)).build());
927            }
928            withErrors.put(p, "WPK");
929        } else if (includeOtherSeverity) {
930            errors.add(TestError.builder(this, Severity.OTHER, INVALID_KEY)
931                    .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
932                    .primitives(p)
933                    .build());
934            withErrors.put(p, "UPK");
935        }
936    }
937
938    private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) {
939        // try to fix common typos and check again if value is still unknown
940        final String harmonizedValue = harmonizeValue(value);
941        if (Utils.isEmpty(harmonizedValue))
942            return;
943        String fixedValue;
944        List<Set<String>> sets = new ArrayList<>();
945        Set<String> presetValues = getPresetValues(key);
946        if (presetValues != null)
947            sets.add(presetValues);
948        Set<String> usedValues = oftenUsedTags.get(key);
949        if (usedValues != null)
950            sets.add(usedValues);
951        fixedValue = sets.stream().anyMatch(possibleValues -> possibleValues.contains(harmonizedValue))
952                ? harmonizedValue : null;
953        if (fixedValue == null && !ignoreForLevenshtein.contains(key)) {
954            int maxPresetValueLen = 0;
955            List<String> fixVals = new ArrayList<>();
956            // use Levenshtein distance to find typical typos
957            int minDist = MAX_LEVENSHTEIN_DISTANCE + 1;
958            for (Set<String> possibleValues: sets) {
959                for (String possibleVal : possibleValues) {
960                    if (possibleVal.isEmpty())
961                        continue;
962                    maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length());
963                    if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) {
964                        // don't suggest fix value when given value is short and lengths are too different
965                        // for example surface=u would result in surface=mud
966                        continue;
967                    }
968                    int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue);
969                    if (dist >= harmonizedValue.length()) {
970                        // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'.
971                        continue;
972                    }
973                    if (dist < minDist) {
974                        minDist = dist;
975                        fixVals.clear();
976                        fixVals.add(possibleVal);
977                    } else if (dist == minDist) {
978                        fixVals.add(possibleVal);
979                    }
980                }
981            }
982            if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE
983                    && !fixVals.isEmpty()
984                    && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) {
985                filterDeprecatedTags(p, key, fixVals);
986                if (!fixVals.isEmpty()) {
987                    if (fixVals.size() < 2) {
988                        fixedValue = fixVals.get(0);
989                    } else {
990                        Collections.sort(fixVals);
991                        // misspelled preset value with multiple good alternatives
992                        errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX)
993                                .message(tr("Unknown property value"),
994                                        marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"),
995                                        value, key, fixVals)
996                                .primitives(p).build());
997                        withErrors.put(p, "WPV");
998                        return;
999                    }
1000                }
1001            }
1002        }
1003        if (fixedValue != null && !fixedValue.equals(value)) {
1004            final String newValue = fixedValue;
1005            // misspelled preset value
1006            errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
1007                    .message(tr("Unknown property value"),
1008                            marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue)
1009                    .primitives(p)
1010                    .build());
1011            withErrors.put(p, "WPV");
1012        } else if (includeOtherSeverity) {
1013            // unknown preset value
1014            errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
1015                    .message(tr("Presets do not contain property value"),
1016                            marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key)
1017                    .primitives(p)
1018                    .build());
1019            withErrors.put(p, "UPV");
1020        }
1021    }
1022
1023    // see #19180
1024    private void filterDeprecatedTags(OsmPrimitive p, String key, List<String> fixVals) {
1025        if (fixVals.isEmpty() || deprecatedChecker == null)
1026            return;
1027
1028        int unchangedDeprecated = countDeprecated(p);
1029
1030        // see #19895: create deep clone. This complex method works even with locked files
1031        MergeSourceBuildingVisitor builder = new MergeSourceBuildingVisitor(p.getDataSet());
1032        p.accept(builder);
1033        DataSet clonedDs = builder.build();
1034        OsmPrimitive clone = clonedDs.getPrimitiveById(p.getPrimitiveId());
1035
1036        Iterator<String> iter = fixVals.iterator();
1037        while (iter.hasNext()) {
1038            clone.put(key, iter.next());
1039            if (countDeprecated(clone) > unchangedDeprecated)
1040                iter.remove();
1041        }
1042    }
1043
1044    private int countDeprecated(OsmPrimitive p) {
1045        if (deprecatedChecker == null)
1046            return 0;
1047        deprecatedChecker.getErrors().clear();
1048        deprecatedChecker.visit(Collections.singleton(p), url -> url.endsWith("deprecated.mapcss"));
1049        return deprecatedChecker.getErrors().size();
1050    }
1051
1052    private static boolean isNum(String harmonizedValue) {
1053        try {
1054            Double.parseDouble(harmonizedValue);
1055            return true;
1056        } catch (NumberFormatException e) {
1057            return false;
1058        }
1059    }
1060
1061    private static boolean isFixme(String key, String value) {
1062        return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
1063          || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
1064    }
1065
1066    private static String harmonizeKey(String key) {
1067        return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
1068    }
1069
1070    private static String harmonizeValue(String value) {
1071        return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
1072    }
1073
1074    @Override
1075    public void startTest(ProgressMonitor monitor) {
1076        super.startTest(monitor);
1077        includeOtherSeverity = includeOtherSeverityChecks();
1078        checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
1079        if (isBeforeUpload) {
1080            checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
1081        }
1082
1083        checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
1084        if (isBeforeUpload) {
1085            checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
1086        }
1087
1088        checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true);
1089        if (isBeforeUpload) {
1090            checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
1091        }
1092
1093        checkFixmes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
1094        if (isBeforeUpload) {
1095            checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
1096        }
1097
1098        checkPresetsTypes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true);
1099        if (isBeforeUpload) {
1100            checkPresetsTypes = checkPresetsTypes && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true);
1101        }
1102        deprecatedChecker = OsmValidator.getTest(MapCSSTagChecker.class);
1103        ignoreForOuterMPSameTagCheck.addAll(Config.getPref().getList(PREF_KEYS_IGNORE_OUTER_MP_SAME_TAG, Collections.emptyList()));
1104    }
1105
1106    @Override
1107    public void endTest() {
1108        deprecatedChecker = null;
1109        super.endTest();
1110    }
1111
1112    @Override
1113    public void visit(Collection<OsmPrimitive> selection) {
1114        if (checkKeys || checkValues || checkComplex || checkFixmes || checkPresetsTypes) {
1115            super.visit(selection);
1116        }
1117    }
1118
1119    @Override
1120    public void addGui(JPanel testPanel) {
1121        GBC a = GBC.eol();
1122        a.anchor = GridBagConstraints.LINE_END;
1123
1124        testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
1125
1126        prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
1127        prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
1128        testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
1129
1130        prefCheckKeysBeforeUpload = new JCheckBox();
1131        prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
1132        testPanel.add(prefCheckKeysBeforeUpload, a);
1133
1134        prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
1135        prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
1136        testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
1137
1138        prefCheckComplexBeforeUpload = new JCheckBox();
1139        prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
1140        testPanel.add(prefCheckComplexBeforeUpload, a);
1141
1142        final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
1143        sourcesList = new EditableList(tr("TagChecker source"));
1144        sourcesList.setItems(sources);
1145        testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
1146        testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
1147
1148        ActionListener disableCheckActionListener = e -> handlePrefEnable();
1149        prefCheckKeys.addActionListener(disableCheckActionListener);
1150        prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
1151        prefCheckComplex.addActionListener(disableCheckActionListener);
1152        prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
1153
1154        handlePrefEnable();
1155
1156        prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
1157        prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
1158        testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
1159
1160        prefCheckValuesBeforeUpload = new JCheckBox();
1161        prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
1162        testPanel.add(prefCheckValuesBeforeUpload, a);
1163
1164        prefCheckFixmes = new JCheckBox(tr("Check for fixme."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
1165        prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with fixme in any property value."));
1166        testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
1167
1168        prefCheckFixmesBeforeUpload = new JCheckBox();
1169        prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
1170        testPanel.add(prefCheckFixmesBeforeUpload, a);
1171
1172        prefCheckPresetsTypes = new JCheckBox(tr("Check for presets types."), Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true));
1173        prefCheckPresetsTypes.setToolTipText(tr("Validate that objects types are valid checking against presets."));
1174        testPanel.add(prefCheckPresetsTypes, GBC.std().insets(20, 0, 0, 0));
1175
1176        prefCheckPresetsTypesBeforeUpload = new JCheckBox();
1177        prefCheckPresetsTypesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true));
1178        testPanel.add(prefCheckPresetsTypesBeforeUpload, a);
1179    }
1180
1181    /**
1182     * Enables/disables the source list field
1183     */
1184    public void handlePrefEnable() {
1185        boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
1186                || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1187        sourcesList.setEnabled(selected);
1188    }
1189
1190    @Override
1191    public boolean ok() {
1192        enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
1193        testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
1194                || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1195
1196        Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
1197        Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
1198        Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
1199        Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
1200        Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES, prefCheckPresetsTypes.isSelected());
1201        Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
1202        Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
1203        Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
1204        Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
1205        Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, prefCheckPresetsTypesBeforeUpload.isSelected());
1206        return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
1207    }
1208
1209    @Override
1210    public Command fixError(TestError testError) {
1211        List<Command> commands = new ArrayList<>(50);
1212
1213        Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
1214        for (OsmPrimitive p : primitives) {
1215            Map<String, String> tags = p.getKeys();
1216            if (tags.isEmpty()) {
1217                continue;
1218            }
1219
1220            for (Entry<String, String> prop: tags.entrySet()) {
1221                String key = prop.getKey();
1222                String value = prop.getValue();
1223                if (Utils.isBlank(value)) {
1224                    commands.add(new ChangePropertyCommand(p, key, null));
1225                } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains("  ")) {
1226                    commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
1227                } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains("  ")) {
1228                    commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
1229                } else {
1230                    String evalue = Entities.unescape(value);
1231                    if (!evalue.equals(value)) {
1232                        commands.add(new ChangePropertyCommand(p, key, evalue));
1233                    }
1234                }
1235            }
1236        }
1237
1238        if (commands.isEmpty())
1239            return null;
1240        if (commands.size() == 1)
1241            return commands.get(0);
1242
1243        return new SequenceCommand(tr("Fix tags"), commands);
1244    }
1245
1246    @Override
1247    public boolean isFixable(TestError testError) {
1248        if (testError.getTester() instanceof TagChecker) {
1249            int code = testError.getCode();
1250            return code == EMPTY_VALUES || code == INVALID_SPACE ||
1251                   code == INVALID_KEY_SPACE || code == INVALID_HTML ||
1252                   code == MULTIPLE_SPACES;
1253        }
1254
1255        return false;
1256    }
1257
1258    @Override
1259    public void taggingPresetsModified() {
1260        try {
1261            initializeData();
1262            initializePresets();
1263            analysePresets();
1264        } catch (IOException e) {
1265            Logging.error(e);
1266        }
1267    }
1268}