Add lint database filtering tool

lint database file (api-versions.xml) is used for NewApi checks. These checks
were written thinking about app developers.

When used inside the platform codebase, quite often it produces false positives
which make relying on it cumbersome.

This CL removes APIs introduced by mainline modules from the lint database to
remove many such false positives. While some of them might be real issues, the
alternative would be not turning these checks on at all.

Bug: 186478867
Bug: 177434707
Test: m lint-check
Test: m filter-api-versions
Test: atest api_versions_trimmer_unittests
Merged-In: Ic45db0b71a2f13dc456463ce04b2f7744a0e330c
Change-Id: Ic45db0b71a2f13dc456463ce04b2f7744a0e330c
This commit is contained in:
Pedro Loureiro 2021-05-26 15:50:07 +00:00 committed by Anton Hansson
parent 33d67f0b14
commit dea35e103b
3 changed files with 512 additions and 0 deletions

View File

@ -24,6 +24,41 @@ package {
default_applicable_licenses: ["frameworks_base_license"], default_applicable_licenses: ["frameworks_base_license"],
} }
python_binary_host {
name: "api_versions_trimmer",
srcs: ["api_versions_trimmer.py"],
version: {
py2: {
enabled: false,
},
py3: {
enabled: true,
embedded_launcher: false,
},
},
}
python_test_host {
name: "api_versions_trimmer_unittests",
main: "api_versions_trimmer_unittests.py",
srcs: [
"api_versions_trimmer_unittests.py",
"api_versions_trimmer.py",
],
test_options: {
unit_test: true,
},
version: {
py2: {
enabled: false,
},
py3: {
enabled: true,
embedded_launcher: false,
},
},
}
metalava_cmd = "$(location metalava)" metalava_cmd = "$(location metalava)"
// Silence reflection warnings. See b/168689341 // Silence reflection warnings. See b/168689341
metalava_cmd += " -J--add-opens=java.base/java.util=ALL-UNNAMED " metalava_cmd += " -J--add-opens=java.base/java.util=ALL-UNNAMED "
@ -401,3 +436,37 @@ genrule {
}, },
], ],
} }
// This rule will filter classes present in the jar files of mainline modules
// from the lint database in api-versions.xml.
// This is done to reduce the number of false positive NewApi findings in
// java libraries that compile against the module SDK
genrule {
name: "api-versions-xml-public-filtered",
srcs: [
// Note: order matters: first parameter is the full api-versions.xml
// after that the stubs files in any order
// stubs files are all modules that export API surfaces EXCEPT ART
":framework-doc-stubs{.api_versions.xml}",
":android.net.ipsec.ike.stubs{.jar}",
":conscrypt.module.public.api.stubs{.jar}",
":framework-connectivity.stubs{.jar}",
":framework-media.stubs{.jar}",
":framework-mediaprovider.stubs{.jar}",
":framework-permission.stubs{.jar}",
":framework-sdkextensions.stubs{.jar}",
":framework-statsd.stubs{.jar}",
":framework-tethering.stubs{.jar}",
":framework-wifi.stubs{.jar}",
":i18n.module.public.api.stubs{.jar}",
],
out: ["api-versions-public-filtered.xml"],
tools: ["api_versions_trimmer"],
cmd: "$(location api_versions_trimmer) $(out) $(in)",
dist: {
targets: [
"sdk",
"win_sdk",
],
},
}

136
api/api_versions_trimmer.py Executable file
View File

@ -0,0 +1,136 @@
#!/usr/bin/env python3
#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Script to remove mainline APIs from the api-versions.xml."""
import argparse
import re
import xml.etree.ElementTree as ET
import zipfile
def read_classes(stubs):
"""Read classes from the stubs file.
Args:
stubs: argument can be a path to a file (a string), a file-like object or a
path-like object
Returns:
a set of the classes found in the file (set of strings)
"""
classes = set()
with zipfile.ZipFile(stubs) as z:
for info in z.infolist():
if (not info.is_dir()
and info.filename.endswith(".class")
and not info.filename.startswith("META-INF")):
# drop ".class" extension
classes.add(info.filename[:-6])
return classes
def filter_method_tag(method, classes_to_remove):
"""Updates the signature of this method by calling filter_method_signature.
Updates the method passed into this function.
Args:
method: xml element that represents a method
classes_to_remove: set of classes you to remove
"""
filtered = filter_method_signature(method.get("name"), classes_to_remove)
method.set("name", filtered)
def filter_method_signature(signature, classes_to_remove):
"""Removes mentions of certain classes from this method signature.
Replaces any existing classes that need to be removed, with java/lang/Object
Args:
signature: string that is a java representation of a method signature
classes_to_remove: set of classes you to remove
"""
regex = re.compile("L.*?;")
start = signature.find("(")
matches = set(regex.findall(signature[start:]))
for m in matches:
# m[1:-1] to drop the leading `L` and `;` ending
if m[1:-1] in classes_to_remove:
signature = signature.replace(m, "Ljava/lang/Object;")
return signature
def filter_lint_database(database, classes_to_remove, output):
"""Reads a lint database and writes a filtered version without some classes.
Reads database from api-versions.xml and removes any references to classes
in the second argument. Writes the result (another xml with the same format
of the database) to output.
Args:
database: path to xml with lint database to read
classes_to_remove: iterable (ideally a set or similar for quick
lookups) that enumerates the classes that should be removed
output: path to write the filtered database
"""
xml = ET.parse(database)
root = xml.getroot()
for c in xml.findall("class"):
cname = c.get("name")
if cname in classes_to_remove:
root.remove(c)
else:
# find the <extends /> tag inside this class to see if the parent
# has been removed from the known classes (attribute called name)
super_classes = c.findall("extends")
for super_class in super_classes:
super_class_name = super_class.get("name")
if super_class_name in classes_to_remove:
super_class.set("name", "java/lang/Object")
interfaces = c.findall("implements")
for interface in interfaces:
interface_name = interface.get("name")
if interface_name in classes_to_remove:
c.remove(interface)
for method in c.findall("method"):
filter_method_tag(method, classes_to_remove)
xml.write(output)
def main():
"""Run the program."""
parser = argparse.ArgumentParser(
description=
("Read a lint database (api-versions.xml) and many stubs jar files. "
"Produce another database file that doesn't include the classes present "
"in the stubs file(s)."))
parser.add_argument("output", help="Destination of the result (xml file).")
parser.add_argument(
"api_versions",
help="The lint database (api-versions.xml file) to read data from"
)
parser.add_argument("stubs", nargs="+", help="The stubs jar file(s)")
parsed = parser.parse_args()
classes = set()
for stub in parsed.stubs:
classes.update(read_classes(stub))
filter_lint_database(parsed.api_versions, classes, parsed.output)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,307 @@
#!/usr/bin/env python3
#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import re
import unittest
import xml.etree.ElementTree as ET
import zipfile
import api_versions_trimmer
def create_in_memory_zip_file(files):
f = io.BytesIO()
with zipfile.ZipFile(f, "w") as z:
for fname in files:
with z.open(fname, mode="w") as class_file:
class_file.write(b"")
return f
def indent(elem, level=0):
i = "\n" + level * " "
j = "\n" + (level - 1) * " "
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for subelem in elem:
indent(subelem, level + 1)
if not elem.tail or not elem.tail.strip():
elem.tail = j
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = j
return elem
def pretty_print(s):
tree = ET.parse(io.StringIO(s))
el = indent(tree.getroot())
res = ET.tostring(el).decode("utf-8")
# remove empty lines inside the result because this still breaks some
# comparisons
return re.sub(r"\n\s*\n", "\n", res, re.MULTILINE)
class ApiVersionsTrimmerUnittests(unittest.TestCase):
def setUp(self):
# so it prints diffs in long strings (xml files)
self.maxDiff = None
def test_read_classes(self):
f = create_in_memory_zip_file(
["a/b/C.class",
"a/b/D.class",
]
)
res = api_versions_trimmer.read_classes(f)
self.assertEqual({"a/b/C", "a/b/D"}, res)
def test_read_classes_ignore_dex(self):
f = create_in_memory_zip_file(
["a/b/C.class",
"a/b/D.class",
"a/b/E.dex",
"f.dex",
]
)
res = api_versions_trimmer.read_classes(f)
self.assertEqual({"a/b/C", "a/b/D"}, res)
def test_read_classes_ignore_manifest(self):
f = create_in_memory_zip_file(
["a/b/C.class",
"a/b/D.class",
"META-INFO/G.class"
]
)
res = api_versions_trimmer.read_classes(f)
self.assertEqual({"a/b/C", "a/b/D"}, res)
def test_filter_method_signature(self):
xml = """
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
"""
method = ET.fromstring(xml)
classes_to_remove = {"android/accessibilityservice/GestureDescription"}
expected = "dispatchGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
api_versions_trimmer.filter_method_tag(method, classes_to_remove)
self.assertEqual(expected, method.get("name"))
def test_filter_method_signature_with_L_in_method(self):
xml = """
<method name="dispatchLeftGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
"""
method = ET.fromstring(xml)
classes_to_remove = {"android/accessibilityservice/GestureDescription"}
expected = "dispatchLeftGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
api_versions_trimmer.filter_method_tag(method, classes_to_remove)
self.assertEqual(expected, method.get("name"))
def test_filter_method_signature_with_L_in_class(self):
xml = """
<method name="dispatchGesture(Landroid/accessibilityservice/LeftGestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
"""
method = ET.fromstring(xml)
classes_to_remove = {"android/accessibilityservice/LeftGestureDescription"}
expected = "dispatchGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
api_versions_trimmer.filter_method_tag(method, classes_to_remove)
self.assertEqual(expected, method.get("name"))
def test_filter_method_signature_with_inner_class(self):
xml = """
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription$Inner;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
"""
method = ET.fromstring(xml)
classes_to_remove = {"android/accessibilityservice/GestureDescription$Inner"}
expected = "dispatchGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
api_versions_trimmer.filter_method_tag(method, classes_to_remove)
self.assertEqual(expected, method.get("name"))
def _run_filter_db_test(self, database_str, expected):
"""Performs the pattern of testing the filter_lint_database method.
Filters instances of the class "a/b/C" (hard-coded) from the database string
and compares the result with the expected result (performs formatting of
the xml of both inputs)
Args:
database_str: string, the contents of the lint database (api-versions.xml)
expected: string, the expected result after filtering the original
database
"""
database = io.StringIO(database_str)
classes_to_remove = {"a/b/C"}
output = io.BytesIO()
api_versions_trimmer.filter_lint_database(
database,
classes_to_remove,
output
)
expected = pretty_print(expected)
res = pretty_print(output.getvalue().decode("utf-8"))
self.assertEqual(expected, res)
def test_filter_lint_database_updates_method_signature_params(self):
self._run_filter_db_test(
database_str="""
<api version="2">
<!-- will be removed -->
<class name="a/b/C" since="1">
<extends name="java/lang/Object"/>
</class>
<class name="a/b/E" since="1">
<!-- extends will be modified -->
<extends name="a/b/C"/>
<!-- first parameter will be modified -->
<method name="dispatchGesture(La/b/C;Landroid/os/Handler;)Z" since="24"/>
<!-- second should remain untouched -->
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
</class>
</api>
""",
expected="""
<api version="2">
<class name="a/b/E" since="1">
<extends name="java/lang/Object"/>
<method name="dispatchGesture(Ljava/lang/Object;Landroid/os/Handler;)Z" since="24"/>
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
</class>
</api>
""")
def test_filter_lint_database_updates_method_signature_return(self):
self._run_filter_db_test(
database_str="""
<api version="2">
<!-- will be removed -->
<class name="a/b/C" since="1">
<extends name="java/lang/Object"/>
</class>
<class name="a/b/E" since="1">
<!-- extends will be modified -->
<extends name="a/b/C"/>
<!-- return type should be changed -->
<method name="gestureIdToString(I)La/b/C;" since="24"/>
</class>
</api>
""",
expected="""
<api version="2">
<class name="a/b/E" since="1">
<extends name="java/lang/Object"/>
<method name="gestureIdToString(I)Ljava/lang/Object;" since="24"/>
</class>
</api>
""")
def test_filter_lint_database_removes_implements(self):
self._run_filter_db_test(
database_str="""
<api version="2">
<!-- will be removed -->
<class name="a/b/C" since="1">
<extends name="java/lang/Object"/>
</class>
<class name="a/b/D" since="1">
<extends name="java/lang/Object"/>
<implements name="a/b/C"/>
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
</class>
</api>
""",
expected="""
<api version="2">
<class name="a/b/D" since="1">
<extends name="java/lang/Object"/>
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
</class>
</api>
""")
def test_filter_lint_database_updates_extends(self):
self._run_filter_db_test(
database_str="""
<api version="2">
<!-- will be removed -->
<class name="a/b/C" since="1">
<extends name="java/lang/Object"/>
</class>
<class name="a/b/E" since="1">
<!-- extends will be modified -->
<extends name="a/b/C"/>
<method name="dispatchGesture(Ljava/lang/Object;Landroid/os/Handler;)Z" since="24"/>
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
</class>
</api>
""",
expected="""
<api version="2">
<class name="a/b/E" since="1">
<extends name="java/lang/Object"/>
<method name="dispatchGesture(Ljava/lang/Object;Landroid/os/Handler;)Z" since="24"/>
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
</class>
</api>
""")
def test_filter_lint_database_removes_class(self):
self._run_filter_db_test(
database_str="""
<api version="2">
<!-- will be removed -->
<class name="a/b/C" since="1">
<extends name="java/lang/Object"/>
</class>
<class name="a/b/D" since="1">
<extends name="java/lang/Object"/>
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
</class>
</api>
""",
expected="""
<api version="2">
<class name="a/b/D" since="1">
<extends name="java/lang/Object"/>
<method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
</class>
</api>
""")
if __name__ == "__main__":
unittest.main()