test/knownfailures.json

[
    {
        "test": "153-reference-stress",
        "description": ["Disable 153-reference-stress temporarily until a fix",
                        "arrives."],
        "bug": "http://b/33389022"
    },
    {
        "test": "080-oom-fragmentation",
        "description": "Disable 080-oom-fragmentation due to flakes.",
        "bug": "http://b/33795328"
    },
    {
        "tests": ["497-inlining-and-class-loader",
                  "542-unresolved-access-check"],
        "description": ["Disable 497-inlining-and-class-loader and ",
                        "542-unresolved-access-check until they are rewritten.",
                        "These tests use a broken class loader that tries to",
                        "register a dex file that's already registered with a",
                        "different loader."],
        "bug": "http://b/34193123"
    },
    {
        "test": "149-suspend-all-stress",
        "description": "Disable 149-suspend-all-stress, its output is flaky",
        "bug": "http://b/28988206"
    },
    {
        "test": "577-profile-foreign-dex",
        "description": "Disable 577-profile-foreign-dex",
        "bug": "http://b/27454772"
    },
    {
        "tests": ["002-sleep",
                  "053-wait-some",
                  "055-enum-performance",
                  "133-static-invoke-super"],
        "description": ["Tests that are timing sensitive and flaky on heavily",
                        "loaded systems."]
    },
    {
        "test": "147-stripped-dex-fallback",
        "variant": "target",
        "description": ["147-stripped-dex-fallback isn't supported on device",
                        "because --strip-dex  requires the zip command."]
    },
    {
        "test": "569-checker-pattern-replacement",
        "variant": "target",
        "description": ["569-checker-pattern-replacement tests behaviour",
                        "present only on host."]
    },
    {
        "tests": ["116-nodex2oat",
                  "118-noimage-dex2oat",
                  "134-nodex2oat-nofallback"],
        "variant": "prebuild",
        "description": ["Note 116-nodex2oat is not broken per-se it just",
                        "doesn't (and isn't meant to) work with --prebuild."]
    },
    {
        "test": "554-jit-profile-file",
        "variant": "no-prebuild | interpreter",
        "description": ["554-jit-profile-file is disabled because it needs a",
                        "primary oat file to know what it should save."]
    },
    {
        "tests": ["529-checker-unresolved", "555-checker-regression-x86const"],
        "variant": "no-prebuild",
        "bug": "http://b/27784033"
    },
    {
        "tests": ["117-nopatchoat",
                  "147-stripped-dex-fallback",
                  "608-checker-unresolved-lse"],
        "variant": "no-prebuild"
    },
    {
        "tests": ["117-nopatchoat",
                  "118-noimage-dex2oat",
                  "119-noimage-patchoat",
                  "554-jit-profile-file"],
        "variant": "no-relocate",
        "description": ["117-nopatchoat is not broken per-se it just doesn't",
                        "work (and isn't meant to) without --prebuild",
                        "--relocate"]
    },
    {
        "test": "137-cfi",
        "variant": "interp-ac",
        "description": ["Temporarily disable some broken tests when forcing",
                        "access checks in interpreter"],
        "bug": "http://b/22414682"
    },
    {
        "test" : "629-vdex-speed",
        "variant": "interp-ac | no-dex2oat | interpreter | jit | relocate-npatchoat",
        "description": "629 requires compilation."
    },
    {
        "test": "137-cfi",
        "variant": "gcstress",
        "description": ["137-cfi needs to unwind a second forked process. We're",
                        "using a primitive sleep to wait till we hope the",
                        "second process got into the expected state. The",
                        "slowness of gcstress makes this bad."]
    },
    {
        "tests": ["908-gc-start-finish",
                  "913-heaps"],
        "variant": "gcstress",
        "description": ["908-gc-start-finish expects GCs only to be run at",
                        "clear points. The reduced heap size makes this",
                        "non-deterministic. Same for 913."]
    },
    {
        "test": "961-default-iface-resolution-gen",
        "variant": "gcstress",
        "description": ["961-default-iface-resolution-gen and",
                        "964-default-iface-init-genare very long tests that",
                        "often will take more than the timeout to run when",
                        "gcstress is enabled. This is because gcstress slows",
                        "down allocations significantly which these tests do a",
                        "lot."]
    },
    {
        "tests": ["964-default-iface-init-gen",
                 "154-gc-loop"],
        "variant": "gcstress"
    },
    {
        "test": "115-native-bridge",
        "variant": "target",
        "description": ["115-native-bridge setup is complicated. Need to",
                        "implement it correctly for the target."]
    },
    {
        "test": "130-hprof",
        "variant": "target",
        "desription": ["130-hprof dumps the heap and runs hprof-conv to check",
                       "whether the file is somewhat readable. Thi is only",
                       "possible on the host. TODO: Turn off all the other",
                       "combinations, this is more about testing actual ART",
                       "code. A gtest is very hard to write here, as (for a",
                       "complete test) JDWP must be set up."]
    },
    {
        "test": "131-structural-change",
        "variant": "debug",
        "description": ["131 is an old test. The functionality has been",
                        "implemented at an earlier stage and is checked",
                        "in tests 138. Blacklisted for debug builds since",
                        "these builds have duplicate classes checks which",
                        "punt to interpreter"]
    },
    {
        "test": "138-duplicate-classes-check",
        "variant": "ndebug",
        "description": ["Turned on for debug builds since debug builds have",
                        "duplicate classes checks enabled"],
        "bug": "http://b/2133391"
    },
    {
        "test": "147-stripped-dex-fallback",
        "variant": "no-dex2oat | no-image | relocate-npatchoat",
        "description": ["147-stripped-dex-fallback is disabled because it",
                        "requires --prebuild."]
    },
    {
        "test": "554-jit-profile-file",
        "variant": "no-dex2oat | no-image | relocate-npatchoat",
        "description": ["554-jit-profile-file is disabled because it needs a",
                        "primary oat file to know what it should save."]
    },
    {
        "tests": ["116-nodex2oat",
                  "117-nopatchoat",
                  "118-noimage-dex2oat",
                  "119-noimage-patchoat",
                  "137-cfi",
                  "138-duplicate-classes-check2"],
        "variant": "no-dex2oat | no-image | relocate-npatchoat",
        "description": ["All these tests check that we have sane behavior if we",
                        "don't have a patchoat or dex2oat. Therefore we",
                        "shouldn't run them in situations where we actually",
                        "don't have these since they explicitly test for them.",
                        "These all also assume we have an image."]
    },
    {
        "tests": ["137-cfi",
                  "138-duplicate-classes-check",
                  "018-stack-overflow",
                  "961-default-iface-resolution-gen",
                  "964-default-iface-init"],
        "variant": "no-image",
        "description": ["This test fails without an image. 018, 961, 964 often",
                        "time out."],
        "bug": "http://b/34369284"
    },
    {
        "test": "137-cfi",
        "description": ["This test unrolls and expects managed frames, but",
                        "tracing means we run the interpreter."],
        "variant": "trace | stream"
    },
    {
        "tests": ["802-deoptimization",
                 "570-checker-osr"],
        "description": ["This test dynamically enables tracing to force a",
                        "deoptimization. This makes the test meaningless",
                        "when already tracing, and writes an error message",
                        "that we do not want to check for."],
        "variant": "trace | stream"
    },
    {
        "test": "130-hprof",
        "description": "130 occasional timeout",
        "bug": "http://b/32383962",
        "variant": "trace | stream"
    },
    {
        "tests": ["087-gc-after-link",
                  "141-class-unload"],
        "variant": "trace | stream"
    },
    {
        "tests": ["604-hot-static-interface",
                  "612-jit-dex-cache",
                  "613-inlining-dex-cache",
                  "616-cha",
                  "626-set-resolved-string"],
        "variant": "trace  | stream",
        "description": ["These tests expect JIT compilation, which is",
                        "suppressed when tracing."]
    },
    {
        "test": "137-cfi",
        "description": ["CFI unwinding expects managed frames, and the test",
                        "does not iterate enough to even compile. JIT also",
                        "uses Generic JNI instead of the JNI compiler."],
        "variant": "interpreter | jit"
    },
    {
        "test": "902-hello-transformation",
        "description": "Test 902 hits races with the JIT compiler.",
        "bug": "http://b/32821077",
        "variant": "jit"
    },
    {
        "test": "906-iterate-heap",
        "description": ["Test 906 iterates the heap filtering with different",
                        "options. No instances should be created between those",
                        "runs to be able to have precise checks."],
        "variant": "jit"
    },
    {
        "tests": ["914-hello-obsolescence",
                  "915-obsolete-2",
                  "917-fields-transformation",
                  "919-obsolete-fields"],
        "description": ["Test 914, 915, 917, & 919 are very sensitive to the",
                        "exact state of the stack, including the jit-inserted",
                        "runtime frames. This causes them to be somewhat flaky",
                        "as JIT tests. This should be fixed once b/33630159 or",
                        "b/33616143 are resolved but until then just disable",
                        "them. Test 916 already checks this feature for JIT",
                        "use cases in a way that is resilient to the jit",
                        "frames."],
        "variant": "jit"
    },
    {
        "tests": ["926-multi-obsolescence",
                  "904-object-allocation"],
        "variant": "jit"
    },
    {
        "test": "912-classes",
        "variant": "jit",
        "bug": "http://b/34655682"
    },
    {
        "tests": ["570-checker-select",
                  "484-checker-register-hints"],
        "description": ["These tests were based on the linear scan allocator,",
                        "which makes different decisions than the graph",
                        "coloring allocator. (These attempt to test for code",
                        "quality, not correctness.)"],
        "variant": "regalloc_gc"
    },
    {
        "tests": ["454-get-vreg",
                  "457-regs",
                  "602-deoptimizeable"],
        "description": ["Tests that should fail when the optimizing compiler ",
                        "compiles them non-debuggable."],
        "variant": "optimizing &  ndebuggable | regalloc_gc & ndebuggable"
    },
    {
        "test": "596-app-images",
        "variant": "npictest"
    },
    {
        "test": "055-enum-performance",
        "variant": "optimizing | regalloc_gc",
        "description": ["055: Exceeds run time limits due to heap poisoning ",
                        "instrumentation (on ARM and ARM64 devices)."]
    },
    {
        "test": "909-attach-agent",
        "variant": "debuggable",
        "description": "Tests that check semantics for a non-debuggable app."
    },
    {
        "test": "137-cfi",
        "variant": "debuggable",
        "description": ["The test relies on AOT code and debuggable makes us",
                        "JIT always."]
    },
    {
        "tests": ["000-nop",
                  "134-nodex2oat-nofallback",
                  "147-stripped-dex-fallback",
                 "595-profile-saving"],
        "description": "The doesn't compile anything",
        "env_vars": {"ART_TEST_BISECTION": "true"},
        "variant": "optimizing | regalloc_gc"
    },
    {
        "tests": "089-many-methods",
        "description": "The test tests a build failure",
        "env_vars": {"ART_TEST_BISECTION": "true"},
        "variant": "optimizing | regalloc_gc"
    },
    {
        "tests": ["018-stack-overflow",
                  "116-nodex2oat",
                  "117-nopatchoat",
                  "118-noimage-dex2oat",
                  "119-noimage-patchoat",
                  "126-miranda-multidex",
                  "137-cfi"],
        "description": "The test run dalvikvm more than once.",
        "env_vars": {"ART_TEST_BISECTION": "true"},
        "variant": "optimizing | regalloc_gc"
    },
    {
        "tests": ["115-native-bridge",
                 "088-monitor-verification"],
        "description": "The test assume they are always compiled.",
        "env_vars": {"ART_TEST_BISECTION": "true"},
        "variant": "optimizing | regalloc_gc"
    },
    {
        "test": "055-enum-performance",
        "description": ["The test tests performance which degrades during",
                        "bisecting."],
        "env_vars": {"ART_TEST_BISECTION": "true"},
        "variant": "optimizing | regalloc_gc"
    },
    {
        "test": "537-checker-arraycopy",
        "env_vars": {"ART_USE_READ_BARRIER": "true"},
        "variant": "interpreter | optimizing | regalloc_gc | jit"
    }
]