001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.awt.geom.Area;
007import java.io.BufferedReader;
008import java.io.File;
009import java.io.IOException;
010import java.io.InputStream;
011import java.io.Reader;
012import java.util.ArrayList;
013import java.util.Collection;
014import java.util.HashMap;
015import java.util.HashSet;
016import java.util.Iterator;
017import java.util.List;
018import java.util.Map;
019import java.util.Map.Entry;
020import java.util.Objects;
021import java.util.Set;
022import java.util.function.Consumer;
023import java.util.function.Predicate;
024import java.util.stream.Stream;
025
026import org.openstreetmap.josm.data.osm.IPrimitive;
027import org.openstreetmap.josm.data.osm.OsmPrimitive;
028import org.openstreetmap.josm.data.preferences.BooleanProperty;
029import org.openstreetmap.josm.data.preferences.CachingProperty;
030import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
031import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
032import org.openstreetmap.josm.data.validation.OsmValidator;
033import org.openstreetmap.josm.data.validation.Severity;
034import org.openstreetmap.josm.data.validation.Test;
035import org.openstreetmap.josm.data.validation.TestError;
036import org.openstreetmap.josm.gui.mappaint.Environment;
037import org.openstreetmap.josm.gui.mappaint.MultiCascade;
038import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
039import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleIndex;
040import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
041import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
042import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
043import org.openstreetmap.josm.gui.progress.ProgressMonitor;
044import org.openstreetmap.josm.io.CachedFile;
045import org.openstreetmap.josm.io.FileWatcher;
046import org.openstreetmap.josm.io.UTFInputStreamReader;
047import org.openstreetmap.josm.spi.preferences.Config;
048import org.openstreetmap.josm.tools.CheckParameterUtil;
049import org.openstreetmap.josm.tools.I18n;
050import org.openstreetmap.josm.tools.Logging;
051import org.openstreetmap.josm.tools.MultiMap;
052import org.openstreetmap.josm.tools.Stopwatch;
053import org.openstreetmap.josm.tools.Utils;
054
055/**
056 * MapCSS-based tag checker/fixer.
057 * @since 6506
058 */
059public class MapCSSTagChecker extends Test.TagTest {
060    private MapCSSStyleIndex indexData;
061    private final Map<MapCSSRule, MapCSSTagCheckerAndRule> ruleToCheckMap = new HashMap<>();
062    private static final Map<IPrimitive, Area> mpAreaCache = new HashMap<>();
063    private static final Set<IPrimitive> toMatchForSurrounding = new HashSet<>();
064    static final boolean ALL_TESTS = true;
065    static final boolean ONLY_SELECTED_TESTS = false;
066
067    /**
068     * Cached version of {@link ValidatorPrefHelper#PREF_OTHER}, see #20745.
069     */
070    private static final CachingProperty<Boolean> PREF_OTHER = new BooleanProperty("validator.other", false).cached();
071
072    /**
073     * The preference key for tag checker source entries.
074     * @since 6670
075     */
076    public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
077
078    /**
079     * Constructs a new {@code MapCSSTagChecker}.
080     */
081    public MapCSSTagChecker() {
082        super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
083    }
084
085    final MultiMap<String, MapCSSTagCheckerRule> checks = new MultiMap<>();
086
087    /** maps the source URL for a test to the title shown in the dialog where known */
088    private final Map<String, String> urlTitles = new HashMap<>();
089
090    /**
091     * Result of {@link MapCSSTagCheckerRule#readMapCSS}
092     * @since 8936
093     */
094    public static class ParseResult {
095        /** Checks successfully parsed */
096        public final List<MapCSSTagCheckerRule> parseChecks;
097        /** Errors that occurred during parsing */
098        public final Collection<Throwable> parseErrors;
099
100        /**
101         * Constructs a new {@code ParseResult}.
102         * @param parseChecks Checks successfully parsed
103         * @param parseErrors Errors that occurred during parsing
104         */
105        public ParseResult(List<MapCSSTagCheckerRule> parseChecks, Collection<Throwable> parseErrors) {
106            this.parseChecks = parseChecks;
107            this.parseErrors = parseErrors;
108        }
109    }
110
111    static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
112        public final MapCSSRule rule;
113        private final MapCSSTagCheckerRule tagCheck;
114        private final String source;
115
116        MapCSSTagCheckerAndRule(MapCSSRule rule) {
117            this.rule = rule;
118            this.tagCheck = null;
119            this.source = "";
120        }
121
122        MapCSSTagCheckerAndRule(MapCSSTagCheckerRule tagCheck, String source) {
123            this.rule = tagCheck.rule;
124            this.tagCheck = tagCheck;
125            this.source = source;
126        }
127
128        @Override
129        public String toString() {
130            return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
131        }
132
133        @Override
134        public String getSource() {
135            return source;
136        }
137    }
138
139    static MapCSSStyleIndex createMapCSSTagCheckerIndex(
140            MultiMap<String, MapCSSTagCheckerRule> checks, boolean includeOtherSeverity, boolean allTests) {
141        final MapCSSStyleIndex index = new MapCSSStyleIndex();
142        final Stream<MapCSSRule> ruleStream = checks.values().stream()
143                .flatMap(Collection::stream)
144                // Ignore "information" level checks if not wanted, unless they also set a MapCSS class
145                .filter(c -> includeOtherSeverity || Severity.OTHER != c.getSeverity() || !c.setClassExpressions.isEmpty())
146                .filter(c -> allTests || c.rule.selectors.stream().anyMatch(Selector.ChildOrParentSelector.class::isInstance))
147                .map(c -> c.rule);
148        index.buildIndex(ruleStream);
149        return index;
150    }
151
152    /**
153     * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
154     * @param p The OSM primitive
155     * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
156     * @return all errors for the given primitive, with or without those of "info" severity
157     */
158    public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
159        final List<TestError> res = new ArrayList<>();
160        if (indexData == null) {
161            indexData = createMapCSSTagCheckerIndex(checks, includeOtherSeverity, ALL_TESTS);
162        }
163
164        final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
165        env.mpAreaCache = mpAreaCache;
166        env.toMatchForSurrounding = toMatchForSurrounding;
167
168        Iterator<MapCSSRule> candidates = indexData.getRuleCandidates(p);
169        while (candidates.hasNext()) {
170            MapCSSRule r = candidates.next();
171            for (Selector selector : r.selectors) {
172                env.clearSelectorMatchingInformation();
173                if (!selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
174                    continue;
175                }
176                MapCSSTagCheckerAndRule test = ruleToCheckMap.computeIfAbsent(r, rule -> checks.entrySet().stream()
177                        .map(e -> e.getValue().stream()
178                                // rule.selectors might be different due to MapCSSStyleIndex, however, the declarations are the same object
179                                .filter(c -> c.rule.declaration == rule.declaration)
180                                .findFirst()
181                                .map(c -> new MapCSSTagCheckerAndRule(c, getTitle(e.getKey())))
182                                .orElse(null))
183                        .filter(Objects::nonNull)
184                        .findFirst()
185                        .orElse(null));
186                MapCSSTagCheckerRule check = test == null ? null : test.tagCheck;
187                if (check != null) {
188                    r.declaration.execute(env);
189                    if (!check.errors.isEmpty()) {
190                        for (TestError e: check.getErrorsForPrimitive(p, selector, env, test)) {
191                            addIfNotSimilar(e, res);
192                        }
193                    }
194                }
195            }
196        }
197        return res;
198    }
199
200    private String getTitle(String url) {
201        return urlTitles.getOrDefault(url, tr("unknown"));
202    }
203
204    /**
205     * See #12627
206     * Add error to given list if list doesn't already contain a similar error.
207     * Similar means same code and description and same combination of primitives and same combination of highlighted objects,
208     * but maybe with different orders.
209     * @param toAdd the error to add
210     * @param errors the list of errors
211     */
212    private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) {
213        final boolean isDup = toAdd.getPrimitives().size() >= 2 && errors.stream().anyMatch(toAdd::isSimilar);
214        if (!isDup)
215            errors.add(toAdd);
216    }
217
218    static Collection<TestError> getErrorsForPrimitive(
219            OsmPrimitive p, boolean includeOtherSeverity, Collection<Set<MapCSSTagCheckerRule>> checksCol) {
220        // this variant is only used by the assertion tests
221        final List<TestError> r = new ArrayList<>();
222        final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
223        env.mpAreaCache = mpAreaCache;
224        env.toMatchForSurrounding = toMatchForSurrounding;
225        for (Set<MapCSSTagCheckerRule> schecks : checksCol) {
226            for (MapCSSTagCheckerRule check : schecks) {
227                boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
228                // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
229                if (ignoreError && check.setClassExpressions.isEmpty()) {
230                    continue;
231                }
232                final Selector selector = check.whichSelectorMatchesEnvironment(env);
233                if (selector != null) {
234                    check.rule.declaration.execute(env);
235                    if (!ignoreError && !check.errors.isEmpty()) {
236                        r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule)));
237                    }
238                }
239            }
240        }
241        return r;
242    }
243
244    /**
245     * Visiting call for primitives.
246     *
247     * @param p The primitive to inspect.
248     */
249    @Override
250    public void check(OsmPrimitive p) {
251        for (TestError e : getErrorsForPrimitive(p, PREF_OTHER.get())) {
252            addIfNotSimilar(e, errors);
253        }
254    }
255
256    /**
257     * A handler for assertion error messages (for not fulfilled "assertMatch", "assertNoMatch").
258     */
259    @FunctionalInterface
260    interface AssertionConsumer extends Consumer<String> {
261    }
262
263    /**
264     * Adds a new MapCSS config file from the given URL.
265     * @param url The unique URL of the MapCSS config file
266     * @return List of tag checks and parsing errors, or null
267     * @throws ParseException if the config file does not match MapCSS syntax
268     * @throws IOException if any I/O error occurs
269     * @since 7275
270     */
271    public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
272        // Check assertions, useful for development of local files
273        final boolean checkAssertions = Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url);
274        return addMapCSS(url, checkAssertions ? Logging::warn : null);
275    }
276
277    synchronized ParseResult addMapCSS(String url, AssertionConsumer assertionConsumer) throws ParseException, IOException {
278        CheckParameterUtil.ensureParameterNotNull(url, "url");
279        ParseResult result;
280        try (CachedFile cache = new CachedFile(url);
281             InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
282             InputStream s = zip != null ? zip : cache.getInputStream();
283             Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
284            if (zip != null)
285                I18n.addTexts(cache.getFile());
286            result = MapCSSTagCheckerRule.readMapCSS(reader, assertionConsumer);
287            checks.remove(url);
288            checks.putAll(url, result.parseChecks);
289            urlTitles.put(url, findURLTitle(url));
290            indexData = null;
291        }
292        return result;
293    }
294
295    /** Find a user friendly string for the url.
296     *
297     * @param url the source for the set of rules
298     * @return a value that can be used in tool tip or progress bar.
299     */
300    private static String findURLTitle(String url) {
301        for (SourceEntry source : new ValidatorPrefHelper().get()) {
302            if (url.equals(source.url) && !Utils.isEmpty(source.title)) {
303                return source.title;
304            }
305        }
306        if (url.endsWith(".mapcss")) // do we have others?
307            url = new File(url).getName();
308        if (url.length() > 33) {
309            url = "..." + url.substring(url.length() - 30);
310        }
311        return url;
312    }
313
314    @Override
315    public synchronized void initialize() throws Exception {
316        checks.clear();
317        urlTitles.clear();
318        indexData = null;
319        for (SourceEntry source : new ValidatorPrefHelper().get()) {
320            if (!source.active) {
321                continue;
322            }
323            String i = source.url;
324            try {
325                if (!i.startsWith("resource:")) {
326                    Logging.info(tr("Adding {0} to tag checker", i));
327                } else if (Logging.isDebugEnabled()) {
328                    Logging.debug(tr("Adding {0} to tag checker", i));
329                }
330                addMapCSS(i);
331                if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
332                    FileWatcher.getDefaultInstance().registerSource(source);
333                }
334            } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
335                Logging.warn(tr("Failed to add {0} to tag checker", i));
336                Logging.log(Logging.LEVEL_WARN, ex);
337            } catch (ParseException | TokenMgrError ex) {
338                Logging.warn(tr("Failed to add {0} to tag checker", i));
339                Logging.warn(ex);
340            }
341        }
342        MapCSSTagCheckerAsserts.clear();
343    }
344
345    /**
346     * Reload tagchecker rule.
347     * @param rule tagchecker rule to reload
348     * @since 12825
349     */
350    public static void reloadRule(SourceEntry rule) {
351        MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
352        if (tagChecker != null) {
353            try {
354                tagChecker.addMapCSS(rule.url);
355            } catch (IOException | ParseException | TokenMgrError e) {
356                Logging.warn(e);
357            }
358        }
359    }
360
361    @Override
362    public synchronized void startTest(ProgressMonitor progressMonitor) {
363        super.startTest(progressMonitor);
364        super.setShowElements(true);
365    }
366
367    @Override
368    public synchronized void endTest() {
369        // no need to keep the index, it is quickly build and doubles the memory needs
370        indexData = null;
371        // always clear the cache to make sure that we catch changes in geometry
372        mpAreaCache.clear();
373        ruleToCheckMap.clear();
374        toMatchForSurrounding.clear();
375        super.endTest();
376    }
377
378    @Override
379    public void visit(Collection<OsmPrimitive> selection) {
380        visit(selection, null);
381    }
382
383    /**
384     * Execute the rules from the URLs matching the given predicate.
385     * @param selection collection of primitives
386     * @param urlPredicate a predicate deciding whether the rules from the given URL shall be executed
387     */
388    void visit(Collection<OsmPrimitive> selection, Predicate<String> urlPredicate) {
389        if (urlPredicate == null && progressMonitor != null) {
390            progressMonitor.setTicksCount(selection.size() * checks.size());
391        }
392
393        mpAreaCache.clear();
394        toMatchForSurrounding.clear();
395
396        Set<OsmPrimitive> surrounding = new HashSet<>();
397        for (Entry<String, Set<MapCSSTagCheckerRule>> entry : checks.entrySet()) {
398            if (isCanceled()) {
399                break;
400            }
401            if (urlPredicate != null && !urlPredicate.test(entry.getKey())) {
402                continue;
403            }
404            visit(entry.getKey(), entry.getValue(), selection, surrounding);
405        }
406    }
407
408    /**
409     * Perform the checks for one check url
410     * @param url the url for the checks
411     * @param checksForUrl the checks to perform
412     * @param selection collection primitives
413     * @param surrounding surrounding primitives, evtl. filled by this routine
414     */
415    private void visit(String url, Set<MapCSSTagCheckerRule> checksForUrl, Collection<OsmPrimitive> selection, Set<OsmPrimitive> surrounding) {
416        MultiMap<String, MapCSSTagCheckerRule> currentCheck = new MultiMap<>();
417        currentCheck.putAll(url, checksForUrl);
418        indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverityChecks(), ALL_TESTS);
419        Set<OsmPrimitive> tested = new HashSet<>();
420
421
422        String title = getTitle(url);
423        if (progressMonitor != null) {
424            progressMonitor.setExtraText(tr(" {0}", title));
425        }
426        long cnt = 0;
427        Stopwatch stopwatch = Stopwatch.createStarted();
428        for (OsmPrimitive p : selection) {
429            if (isCanceled()) {
430                break;
431            }
432            if (isPrimitiveUsable(p)) {
433                check(p);
434                if (partialSelection) {
435                    tested.add(p);
436                }
437            }
438            if (progressMonitor != null) {
439                progressMonitor.worked(1);
440                cnt++;
441                // add frequently changing info to progress monitor so that it
442                // doesn't seem to hang when test takes longer than 0.5 seconds
443                if (cnt % 10000 == 0 && stopwatch.elapsed() >= 500) {
444                    progressMonitor.setExtraText(tr(" {0}: {1} of {2} elements done", title, cnt, selection.size()));
445                }
446            }
447        }
448
449        if (partialSelection && !tested.isEmpty()) {
450            testPartial(currentCheck, tested, surrounding);
451        }
452    }
453
454    private void testPartial(MultiMap<String, MapCSSTagCheckerRule> currentCheck, Set<OsmPrimitive> tested, Set<OsmPrimitive> surrounding) {
455
456        // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15
457        // execute tests for objects which might contain or cross previously tested elements
458
459        final boolean includeOtherSeverity = includeOtherSeverityChecks();
460        // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors
461        // matching the previously tested elements
462        indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverity, ONLY_SELECTED_TESTS);
463        if (indexData.isEmpty())
464            return; // performance: some *.mapcss rule files don't use ChildOrParentSelector
465
466        if (surrounding.isEmpty()) {
467            for (OsmPrimitive p : tested) {
468                if (p.getDataSet() != null) {
469                    surrounding.addAll(p.getDataSet().searchWays(p.getBBox()));
470                    surrounding.addAll(p.getDataSet().searchRelations(p.getBBox()));
471                }
472            }
473        }
474
475        toMatchForSurrounding.clear();
476        toMatchForSurrounding.addAll(tested);
477        for (OsmPrimitive p : surrounding) {
478            if (tested.contains(p))
479                continue;
480            Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity);
481            for (TestError e : additionalErrors) {
482                if (e.getPrimitives().stream().anyMatch(tested::contains))
483                    addIfNotSimilar(e, errors);
484            }
485        }
486
487    }
488
489}