001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.awt.geom.Area; 007import java.io.BufferedReader; 008import java.io.File; 009import java.io.IOException; 010import java.io.InputStream; 011import java.io.Reader; 012import java.util.ArrayList; 013import java.util.Collection; 014import java.util.HashMap; 015import java.util.HashSet; 016import java.util.Iterator; 017import java.util.List; 018import java.util.Map; 019import java.util.Map.Entry; 020import java.util.Objects; 021import java.util.Set; 022import java.util.function.Consumer; 023import java.util.function.Predicate; 024import java.util.stream.Stream; 025 026import org.openstreetmap.josm.data.osm.IPrimitive; 027import org.openstreetmap.josm.data.osm.OsmPrimitive; 028import org.openstreetmap.josm.data.preferences.BooleanProperty; 029import org.openstreetmap.josm.data.preferences.CachingProperty; 030import org.openstreetmap.josm.data.preferences.sources.SourceEntry; 031import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 032import org.openstreetmap.josm.data.validation.OsmValidator; 033import org.openstreetmap.josm.data.validation.Severity; 034import org.openstreetmap.josm.data.validation.Test; 035import org.openstreetmap.josm.data.validation.TestError; 036import org.openstreetmap.josm.gui.mappaint.Environment; 037import org.openstreetmap.josm.gui.mappaint.MultiCascade; 038import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule; 039import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleIndex; 040import org.openstreetmap.josm.gui.mappaint.mapcss.Selector; 041import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException; 042import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError; 043import org.openstreetmap.josm.gui.progress.ProgressMonitor; 044import org.openstreetmap.josm.io.CachedFile; 045import org.openstreetmap.josm.io.FileWatcher; 046import org.openstreetmap.josm.io.UTFInputStreamReader; 047import org.openstreetmap.josm.spi.preferences.Config; 048import org.openstreetmap.josm.tools.CheckParameterUtil; 049import org.openstreetmap.josm.tools.I18n; 050import org.openstreetmap.josm.tools.Logging; 051import org.openstreetmap.josm.tools.MultiMap; 052import org.openstreetmap.josm.tools.Stopwatch; 053import org.openstreetmap.josm.tools.Utils; 054 055/** 056 * MapCSS-based tag checker/fixer. 057 * @since 6506 058 */ 059public class MapCSSTagChecker extends Test.TagTest { 060 private MapCSSStyleIndex indexData; 061 private final Map<MapCSSRule, MapCSSTagCheckerAndRule> ruleToCheckMap = new HashMap<>(); 062 private static final Map<IPrimitive, Area> mpAreaCache = new HashMap<>(); 063 private static final Set<IPrimitive> toMatchForSurrounding = new HashSet<>(); 064 static final boolean ALL_TESTS = true; 065 static final boolean ONLY_SELECTED_TESTS = false; 066 067 /** 068 * Cached version of {@link ValidatorPrefHelper#PREF_OTHER}, see #20745. 069 */ 070 private static final CachingProperty<Boolean> PREF_OTHER = new BooleanProperty("validator.other", false).cached(); 071 072 /** 073 * The preference key for tag checker source entries. 074 * @since 6670 075 */ 076 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries"; 077 078 /** 079 * Constructs a new {@code MapCSSTagChecker}. 080 */ 081 public MapCSSTagChecker() { 082 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values.")); 083 } 084 085 final MultiMap<String, MapCSSTagCheckerRule> checks = new MultiMap<>(); 086 087 /** maps the source URL for a test to the title shown in the dialog where known */ 088 private final Map<String, String> urlTitles = new HashMap<>(); 089 090 /** 091 * Result of {@link MapCSSTagCheckerRule#readMapCSS} 092 * @since 8936 093 */ 094 public static class ParseResult { 095 /** Checks successfully parsed */ 096 public final List<MapCSSTagCheckerRule> parseChecks; 097 /** Errors that occurred during parsing */ 098 public final Collection<Throwable> parseErrors; 099 100 /** 101 * Constructs a new {@code ParseResult}. 102 * @param parseChecks Checks successfully parsed 103 * @param parseErrors Errors that occurred during parsing 104 */ 105 public ParseResult(List<MapCSSTagCheckerRule> parseChecks, Collection<Throwable> parseErrors) { 106 this.parseChecks = parseChecks; 107 this.parseErrors = parseErrors; 108 } 109 } 110 111 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker { 112 public final MapCSSRule rule; 113 private final MapCSSTagCheckerRule tagCheck; 114 private final String source; 115 116 MapCSSTagCheckerAndRule(MapCSSRule rule) { 117 this.rule = rule; 118 this.tagCheck = null; 119 this.source = ""; 120 } 121 122 MapCSSTagCheckerAndRule(MapCSSTagCheckerRule tagCheck, String source) { 123 this.rule = tagCheck.rule; 124 this.tagCheck = tagCheck; 125 this.source = source; 126 } 127 128 @Override 129 public String toString() { 130 return "MapCSSTagCheckerAndRule [rule=" + rule + ']'; 131 } 132 133 @Override 134 public String getSource() { 135 return source; 136 } 137 } 138 139 static MapCSSStyleIndex createMapCSSTagCheckerIndex( 140 MultiMap<String, MapCSSTagCheckerRule> checks, boolean includeOtherSeverity, boolean allTests) { 141 final MapCSSStyleIndex index = new MapCSSStyleIndex(); 142 final Stream<MapCSSRule> ruleStream = checks.values().stream() 143 .flatMap(Collection::stream) 144 // Ignore "information" level checks if not wanted, unless they also set a MapCSS class 145 .filter(c -> includeOtherSeverity || Severity.OTHER != c.getSeverity() || !c.setClassExpressions.isEmpty()) 146 .filter(c -> allTests || c.rule.selectors.stream().anyMatch(Selector.ChildOrParentSelector.class::isInstance)) 147 .map(c -> c.rule); 148 index.buildIndex(ruleStream); 149 return index; 150 } 151 152 /** 153 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}. 154 * @param p The OSM primitive 155 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned 156 * @return all errors for the given primitive, with or without those of "info" severity 157 */ 158 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) { 159 final List<TestError> res = new ArrayList<>(); 160 if (indexData == null) { 161 indexData = createMapCSSTagCheckerIndex(checks, includeOtherSeverity, ALL_TESTS); 162 } 163 164 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 165 env.mpAreaCache = mpAreaCache; 166 env.toMatchForSurrounding = toMatchForSurrounding; 167 168 Iterator<MapCSSRule> candidates = indexData.getRuleCandidates(p); 169 while (candidates.hasNext()) { 170 MapCSSRule r = candidates.next(); 171 for (Selector selector : r.selectors) { 172 env.clearSelectorMatchingInformation(); 173 if (!selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector) 174 continue; 175 } 176 MapCSSTagCheckerAndRule test = ruleToCheckMap.computeIfAbsent(r, rule -> checks.entrySet().stream() 177 .map(e -> e.getValue().stream() 178 // rule.selectors might be different due to MapCSSStyleIndex, however, the declarations are the same object 179 .filter(c -> c.rule.declaration == rule.declaration) 180 .findFirst() 181 .map(c -> new MapCSSTagCheckerAndRule(c, getTitle(e.getKey()))) 182 .orElse(null)) 183 .filter(Objects::nonNull) 184 .findFirst() 185 .orElse(null)); 186 MapCSSTagCheckerRule check = test == null ? null : test.tagCheck; 187 if (check != null) { 188 r.declaration.execute(env); 189 if (!check.errors.isEmpty()) { 190 for (TestError e: check.getErrorsForPrimitive(p, selector, env, test)) { 191 addIfNotSimilar(e, res); 192 } 193 } 194 } 195 } 196 } 197 return res; 198 } 199 200 private String getTitle(String url) { 201 return urlTitles.getOrDefault(url, tr("unknown")); 202 } 203 204 /** 205 * See #12627 206 * Add error to given list if list doesn't already contain a similar error. 207 * Similar means same code and description and same combination of primitives and same combination of highlighted objects, 208 * but maybe with different orders. 209 * @param toAdd the error to add 210 * @param errors the list of errors 211 */ 212 private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) { 213 final boolean isDup = toAdd.getPrimitives().size() >= 2 && errors.stream().anyMatch(toAdd::isSimilar); 214 if (!isDup) 215 errors.add(toAdd); 216 } 217 218 static Collection<TestError> getErrorsForPrimitive( 219 OsmPrimitive p, boolean includeOtherSeverity, Collection<Set<MapCSSTagCheckerRule>> checksCol) { 220 // this variant is only used by the assertion tests 221 final List<TestError> r = new ArrayList<>(); 222 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 223 env.mpAreaCache = mpAreaCache; 224 env.toMatchForSurrounding = toMatchForSurrounding; 225 for (Set<MapCSSTagCheckerRule> schecks : checksCol) { 226 for (MapCSSTagCheckerRule check : schecks) { 227 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity; 228 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class 229 if (ignoreError && check.setClassExpressions.isEmpty()) { 230 continue; 231 } 232 final Selector selector = check.whichSelectorMatchesEnvironment(env); 233 if (selector != null) { 234 check.rule.declaration.execute(env); 235 if (!ignoreError && !check.errors.isEmpty()) { 236 r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule))); 237 } 238 } 239 } 240 } 241 return r; 242 } 243 244 /** 245 * Visiting call for primitives. 246 * 247 * @param p The primitive to inspect. 248 */ 249 @Override 250 public void check(OsmPrimitive p) { 251 for (TestError e : getErrorsForPrimitive(p, PREF_OTHER.get())) { 252 addIfNotSimilar(e, errors); 253 } 254 } 255 256 /** 257 * A handler for assertion error messages (for not fulfilled "assertMatch", "assertNoMatch"). 258 */ 259 @FunctionalInterface 260 interface AssertionConsumer extends Consumer<String> { 261 } 262 263 /** 264 * Adds a new MapCSS config file from the given URL. 265 * @param url The unique URL of the MapCSS config file 266 * @return List of tag checks and parsing errors, or null 267 * @throws ParseException if the config file does not match MapCSS syntax 268 * @throws IOException if any I/O error occurs 269 * @since 7275 270 */ 271 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException { 272 // Check assertions, useful for development of local files 273 final boolean checkAssertions = Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url); 274 return addMapCSS(url, checkAssertions ? Logging::warn : null); 275 } 276 277 synchronized ParseResult addMapCSS(String url, AssertionConsumer assertionConsumer) throws ParseException, IOException { 278 CheckParameterUtil.ensureParameterNotNull(url, "url"); 279 ParseResult result; 280 try (CachedFile cache = new CachedFile(url); 281 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", ""); 282 InputStream s = zip != null ? zip : cache.getInputStream(); 283 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) { 284 if (zip != null) 285 I18n.addTexts(cache.getFile()); 286 result = MapCSSTagCheckerRule.readMapCSS(reader, assertionConsumer); 287 checks.remove(url); 288 checks.putAll(url, result.parseChecks); 289 urlTitles.put(url, findURLTitle(url)); 290 indexData = null; 291 } 292 return result; 293 } 294 295 /** Find a user friendly string for the url. 296 * 297 * @param url the source for the set of rules 298 * @return a value that can be used in tool tip or progress bar. 299 */ 300 private static String findURLTitle(String url) { 301 for (SourceEntry source : new ValidatorPrefHelper().get()) { 302 if (url.equals(source.url) && !Utils.isEmpty(source.title)) { 303 return source.title; 304 } 305 } 306 if (url.endsWith(".mapcss")) // do we have others? 307 url = new File(url).getName(); 308 if (url.length() > 33) { 309 url = "..." + url.substring(url.length() - 30); 310 } 311 return url; 312 } 313 314 @Override 315 public synchronized void initialize() throws Exception { 316 checks.clear(); 317 urlTitles.clear(); 318 indexData = null; 319 for (SourceEntry source : new ValidatorPrefHelper().get()) { 320 if (!source.active) { 321 continue; 322 } 323 String i = source.url; 324 try { 325 if (!i.startsWith("resource:")) { 326 Logging.info(tr("Adding {0} to tag checker", i)); 327 } else if (Logging.isDebugEnabled()) { 328 Logging.debug(tr("Adding {0} to tag checker", i)); 329 } 330 addMapCSS(i); 331 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) { 332 FileWatcher.getDefaultInstance().registerSource(source); 333 } 334 } catch (IOException | IllegalStateException | IllegalArgumentException ex) { 335 Logging.warn(tr("Failed to add {0} to tag checker", i)); 336 Logging.log(Logging.LEVEL_WARN, ex); 337 } catch (ParseException | TokenMgrError ex) { 338 Logging.warn(tr("Failed to add {0} to tag checker", i)); 339 Logging.warn(ex); 340 } 341 } 342 MapCSSTagCheckerAsserts.clear(); 343 } 344 345 /** 346 * Reload tagchecker rule. 347 * @param rule tagchecker rule to reload 348 * @since 12825 349 */ 350 public static void reloadRule(SourceEntry rule) { 351 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class); 352 if (tagChecker != null) { 353 try { 354 tagChecker.addMapCSS(rule.url); 355 } catch (IOException | ParseException | TokenMgrError e) { 356 Logging.warn(e); 357 } 358 } 359 } 360 361 @Override 362 public synchronized void startTest(ProgressMonitor progressMonitor) { 363 super.startTest(progressMonitor); 364 super.setShowElements(true); 365 } 366 367 @Override 368 public synchronized void endTest() { 369 // no need to keep the index, it is quickly build and doubles the memory needs 370 indexData = null; 371 // always clear the cache to make sure that we catch changes in geometry 372 mpAreaCache.clear(); 373 ruleToCheckMap.clear(); 374 toMatchForSurrounding.clear(); 375 super.endTest(); 376 } 377 378 @Override 379 public void visit(Collection<OsmPrimitive> selection) { 380 visit(selection, null); 381 } 382 383 /** 384 * Execute the rules from the URLs matching the given predicate. 385 * @param selection collection of primitives 386 * @param urlPredicate a predicate deciding whether the rules from the given URL shall be executed 387 */ 388 void visit(Collection<OsmPrimitive> selection, Predicate<String> urlPredicate) { 389 if (urlPredicate == null && progressMonitor != null) { 390 progressMonitor.setTicksCount(selection.size() * checks.size()); 391 } 392 393 mpAreaCache.clear(); 394 toMatchForSurrounding.clear(); 395 396 Set<OsmPrimitive> surrounding = new HashSet<>(); 397 for (Entry<String, Set<MapCSSTagCheckerRule>> entry : checks.entrySet()) { 398 if (isCanceled()) { 399 break; 400 } 401 if (urlPredicate != null && !urlPredicate.test(entry.getKey())) { 402 continue; 403 } 404 visit(entry.getKey(), entry.getValue(), selection, surrounding); 405 } 406 } 407 408 /** 409 * Perform the checks for one check url 410 * @param url the url for the checks 411 * @param checksForUrl the checks to perform 412 * @param selection collection primitives 413 * @param surrounding surrounding primitives, evtl. filled by this routine 414 */ 415 private void visit(String url, Set<MapCSSTagCheckerRule> checksForUrl, Collection<OsmPrimitive> selection, Set<OsmPrimitive> surrounding) { 416 MultiMap<String, MapCSSTagCheckerRule> currentCheck = new MultiMap<>(); 417 currentCheck.putAll(url, checksForUrl); 418 indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverityChecks(), ALL_TESTS); 419 Set<OsmPrimitive> tested = new HashSet<>(); 420 421 422 String title = getTitle(url); 423 if (progressMonitor != null) { 424 progressMonitor.setExtraText(tr(" {0}", title)); 425 } 426 long cnt = 0; 427 Stopwatch stopwatch = Stopwatch.createStarted(); 428 for (OsmPrimitive p : selection) { 429 if (isCanceled()) { 430 break; 431 } 432 if (isPrimitiveUsable(p)) { 433 check(p); 434 if (partialSelection) { 435 tested.add(p); 436 } 437 } 438 if (progressMonitor != null) { 439 progressMonitor.worked(1); 440 cnt++; 441 // add frequently changing info to progress monitor so that it 442 // doesn't seem to hang when test takes longer than 0.5 seconds 443 if (cnt % 10000 == 0 && stopwatch.elapsed() >= 500) { 444 progressMonitor.setExtraText(tr(" {0}: {1} of {2} elements done", title, cnt, selection.size())); 445 } 446 } 447 } 448 449 if (partialSelection && !tested.isEmpty()) { 450 testPartial(currentCheck, tested, surrounding); 451 } 452 } 453 454 private void testPartial(MultiMap<String, MapCSSTagCheckerRule> currentCheck, Set<OsmPrimitive> tested, Set<OsmPrimitive> surrounding) { 455 456 // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15 457 // execute tests for objects which might contain or cross previously tested elements 458 459 final boolean includeOtherSeverity = includeOtherSeverityChecks(); 460 // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors 461 // matching the previously tested elements 462 indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverity, ONLY_SELECTED_TESTS); 463 if (indexData.isEmpty()) 464 return; // performance: some *.mapcss rule files don't use ChildOrParentSelector 465 466 if (surrounding.isEmpty()) { 467 for (OsmPrimitive p : tested) { 468 if (p.getDataSet() != null) { 469 surrounding.addAll(p.getDataSet().searchWays(p.getBBox())); 470 surrounding.addAll(p.getDataSet().searchRelations(p.getBBox())); 471 } 472 } 473 } 474 475 toMatchForSurrounding.clear(); 476 toMatchForSurrounding.addAll(tested); 477 for (OsmPrimitive p : surrounding) { 478 if (tested.contains(p)) 479 continue; 480 Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity); 481 for (TestError e : additionalErrors) { 482 if (e.getPrimitives().stream().anyMatch(tested::contains)) 483 addIfNotSimilar(e, errors); 484 } 485 } 486 487 } 488 489}