drgreen/android_frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement Pii Stripper Part 3

Browse Source 
The incident request args sets privacy spec. Strip action is optimized
to run once for each type of spec and ready for flush multiple times.
Incident command is updated to take -p option to specify privacy spec.

Bug: 64687253
Test: unit tests written, manually run incident command to test as well
Change-Id: I6753df117f76dc1a5f4d2152baa3fbbf56b490e4
This commit is contained in:
Yi Jin
2017-09-05 13:44:22 -07:00
parent 99c248feb2
commit 0f0471623e
12 changed files with 312 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
cmds/incident/main.cpp
View File

@ -25,6 +25,7 @@
#include <binder/IServiceManager.h>
#include <utils/Looper.h>
#include <cstring>
#include <fcntl.h>
#include <getopt.h>
#include <stdio.h>
@ -143,6 +144,16 @@ find_section(const char* name)
return NULL;
}
// ================================================================================
static int
get_dest(const char* arg)
{
if (strcmp(arg, "LOCAL") == 0) return 0;
if (strcmp(arg, "EXPLICIT") == 0) return 1;
if (strcmp(arg, "AUTOMATIC") == 0) return 2;
return -1; // return the default value
}
// ================================================================================
static void
usage(FILE* out)
@ -155,6 +166,7 @@ usage(FILE* out)
fprintf(out, " -b (default) print the report to stdout (in proto format)\n");
fprintf(out, " -d send the report into dropbox\n");
fprintf(out, " -l list available sections\n");
fprintf(out, " -p privacy spec, LOCAL, EXPLICIT or AUTOMATIC\n");
fprintf(out, "\n");
fprintf(out, " SECTION the field numbers of the incident report fields to include\n");
fprintf(out, "\n");
@ -166,10 +178,11 @@ main(int argc, char** argv)
Status status;
IncidentReportArgs args;
enum { DEST_DROPBOX, DEST_STDOUT } destination = DEST_STDOUT;
int dest = -1; // default
// Parse the args
int opt;
while ((opt = getopt(argc, argv, "bhdl")) != -1) {
while ((opt = getopt(argc, argv, "bhdlp:")) != -1) {
switch (opt) {
case 'h':
usage(stdout);
@ -183,6 +196,9 @@ main(int argc, char** argv)
case 'd':
destination = DEST_DROPBOX;
break;
case 'p':
dest = get_dest(optarg);
break;
default:
usage(stderr);
return 1;
@ -210,8 +226,7 @@ main(int argc, char** argv)
}
}
}
args.setDest(dest);
// Start the thread pool.
sp<ProcessState> ps(ProcessState::self());

4
cmds/incidentd/src/EncodedBuffer.cpp
View File

@ -70,7 +70,6 @@ write_field_or_skip(FdBuffer::iterator &iterator, vector<uint8_t> &buf, uint8_t
if (skip) {
iterator += bytesToWrite;
} else {
buf.reserve(bytesToWrite);
for (size_t i=0; i<bytesToWrite; i++) {
buf.push_back(*iterator);
iterator++;
@ -192,4 +191,5 @@ EncodedBuffer::flush(int fd)
if (err != NO_ERROR) return err;
}
return NO_ERROR;
}
}

59
cmds/incidentd/src/FdBuffer.cpp
View File

@ -258,6 +258,12 @@ FdBuffer::flush(int fd) const
return write_all(fd, mBuffers[i], mCurrentWritten);
}
FdBuffer::iterator
FdBuffer::begin() const
{
return iterator(*this, 0, 0);
}
FdBuffer::iterator
FdBuffer::end() const
{
@ -268,6 +274,17 @@ FdBuffer::end() const
return FdBuffer::iterator(*this, mBuffers.size() - 1, mCurrentWritten);
}
// ===============================================================================
FdBuffer::iterator::iterator(const FdBuffer& buffer, ssize_t index, ssize_t offset)
: mFdBuffer(buffer),
mIndex(index),
mOffset(offset)
{
}
FdBuffer::iterator&
FdBuffer::iterator::operator=(iterator& other) const { return other; }
FdBuffer::iterator&
FdBuffer::iterator::operator+(size_t offset)
{
@ -280,8 +297,50 @@ FdBuffer::iterator::operator+(size_t offset)
return *this;
}
FdBuffer::iterator&
FdBuffer::iterator::operator+=(size_t offset) { return *this + offset; }
FdBuffer::iterator&
FdBuffer::iterator::operator++() { return *this + 1; }
FdBuffer::iterator
FdBuffer::iterator::operator++(int) { return *this + 1; }
bool
FdBuffer::iterator::operator==(iterator other) const
{
return mIndex == other.mIndex && mOffset == other.mOffset;
}
bool
FdBuffer::iterator::operator!=(iterator other) const { return !(*this == other); }
int
FdBuffer::iterator::operator-(iterator other) const
{
return (int)bytesRead() - (int)other.bytesRead();
}
FdBuffer::iterator::reference
FdBuffer::iterator::operator*() const
{
return mFdBuffer.mBuffers[mIndex][mOffset];
}
FdBuffer::iterator
FdBuffer::iterator::snapshot() const
{
return FdBuffer::iterator(mFdBuffer, mIndex, mOffset);
}
size_t
FdBuffer::iterator::bytesRead() const
{
return mIndex * BUFFER_SIZE + mOffset;
}
bool
FdBuffer::iterator::outOfBound() const
{
return bytesRead() > mFdBuffer.size();
}

29
cmds/incidentd/src/FdBuffer.h
View File

@ -87,32 +87,29 @@ public:
friend class iterator;
class iterator : public std::iterator<std::random_access_iterator_tag, uint8_t> {
public:
iterator(const FdBuffer& buffer, ssize_t index, ssize_t offset)
: mFdBuffer(buffer), mIndex(index), mOffset(offset) {}
iterator& operator=(iterator& other) const { return other; }
iterator& operator+(size_t offset); // this is implemented in .cpp
iterator& operator+=(size_t offset) { return *this + offset; }
iterator& operator++() { return *this + 1; }
iterator operator++(int) { return *this + 1; }
bool operator==(iterator other) const {
return mIndex == other.mIndex && mOffset == other.mOffset;
}
bool operator!=(iterator other) const { return !(*this == other); }
int operator-(iterator other) const { return (int)bytesRead() - (int)other.bytesRead(); }
reference operator*() const { return mFdBuffer.mBuffers[mIndex][mOffset]; }
iterator(const FdBuffer& buffer, ssize_t index, ssize_t offset);
iterator& operator=(iterator& other) const;
iterator& operator+(size_t offset);
iterator& operator+=(size_t offset);
iterator& operator++();
iterator operator++(int);
bool operator==(iterator other) const;
bool operator!=(iterator other) const;
int operator-(iterator other) const;
reference operator*() const;
// return the snapshot of the current iterator
iterator snapshot() const { return iterator(mFdBuffer, mIndex, mOffset); }
iterator snapshot() const;
// how many bytes are read
size_t bytesRead() const;
// random access could make the iterator out of bound
bool outOfBound() const { return bytesRead() > mFdBuffer.size(); }
bool outOfBound() const;
private:
const FdBuffer& mFdBuffer;
size_t mIndex;
size_t mOffset;
};
iterator begin() const { return iterator(*this, 0, 0); }
iterator begin() const;
iterator end() const;
private:

11
cmds/incidentd/src/Privacy.cpp
View File

@ -87,6 +87,12 @@ static bool allowDest(const uint8_t dest, const uint8_t policy)
}
}
bool
PrivacySpec::operator<(const PrivacySpec& other) const
{
return dest < other.dest;
}
bool
PrivacySpec::CheckPremission(const Privacy* privacy) const
{
@ -97,4 +103,9 @@ PrivacySpec::CheckPremission(const Privacy* privacy) const
bool
PrivacySpec::RequireAll() const { return dest == DEST_LOCAL; }
PrivacySpec new_spec_from_args(int dest) {
if (dest < 0) return PrivacySpec();
return PrivacySpec(dest);
}
PrivacySpec get_default_dropbox_spec() { return PrivacySpec(DEST_AUTOMATIC); }

3
cmds/incidentd/src/Privacy.h
View File

@ -58,10 +58,13 @@ public:
PrivacySpec() : dest(DEST_DEFAULT_VALUE) {}
PrivacySpec(uint8_t dest) : dest(dest) {}
bool operator<(const PrivacySpec& other) const;
bool CheckPremission(const Privacy* privacy) const;
bool RequireAll() const;
};
PrivacySpec new_spec_from_args(int dest);
PrivacySpec get_default_dropbox_spec();
#endif // PRIVACY_H

50
cmds/incidentd/src/Section.cpp
View File

@ -27,6 +27,7 @@
#include <private/android_filesystem_config.h>
#include <binder/IServiceManager.h>
#include <map>
#include <mutex>
#include <wait.h>
#include <unistd.h>
@ -109,20 +110,16 @@ GetPrivacyOfSection(int id)
}
static status_t
WriteToRequest(const int id, const int fd, EncodedBuffer& buffer, const PrivacySpec& spec)
WriteToRequest(const int id, const int fd, EncodedBuffer& buffer)
{
if (fd < 0) return EBADF;
if (buffer.size() == 0) return NO_ERROR;
status_t err = NO_ERROR;
uint8_t buf[20];
buffer.clear(); // clear before strip
err = buffer.strip(spec); // TODO: don't have to strip again if the spec is the same.
if (err != NO_ERROR || buffer.size() == 0) return err;
uint8_t *p = write_length_delimited_tag_header(buf, id, buffer.size());
err = write_all(fd, buf, p-buf);
if (err == NO_ERROR) {
err = buffer.flush(fd);
ALOGD("Section %d flushed %zu bytes to fd %d with spec %d", id, buffer.size(), fd, spec.dest);
}
return err;
}
@ -130,25 +127,46 @@ WriteToRequest(const int id, const int fd, EncodedBuffer& buffer, const PrivacyS
static status_t
WriteToReportRequests(const int id, const FdBuffer& buffer, ReportRequestSet* requests)
{
status_t err = EBADF;
status_t err = -EBADF;
EncodedBuffer encodedBuffer(buffer, GetPrivacyOfSection(id));
int writeable = 0;
// The streaming ones
// The streaming ones, group requests by spec in order to save unnecessary strip operations
map<PrivacySpec, vector<sp<ReportRequest>>> requestsBySpec;
for (ReportRequestSet::iterator it = requests->begin(); it != requests->end(); it++) {
sp<ReportRequest> request = *it;
PrivacySpec spec; // TODO: this should be derived from each request.
err = WriteToRequest(id, request->fd, encodedBuffer, spec);
if (err != NO_ERROR) {
request->err = err;
} else {
writeable++;
if (!request->args.containsSection(id) || request->fd < 0 || request->err != NO_ERROR) {
continue; // skip invalid request
}
PrivacySpec spec = new_spec_from_args(request->args.dest());
requestsBySpec[spec].push_back(request);
}
for (map<PrivacySpec, vector<sp<ReportRequest>>>::iterator mit = requestsBySpec.begin(); mit != requestsBySpec.end(); mit++) {
PrivacySpec spec = mit->first;
err = encodedBuffer.strip(spec);
if (err != NO_ERROR) return err; // it means the encodedBuffer data is corrupted.
if (encodedBuffer.size() == 0) continue;
for (vector<sp<ReportRequest>>::iterator it = mit->second.begin(); it != mit->second.end(); it++) {
sp<ReportRequest> request = *it;
err = WriteToRequest(id, request->fd, encodedBuffer);
if (err != NO_ERROR) {
request->err = err;
} else {
writeable++;
}
ALOGD("Section %d flushed %zu bytes to fd %d with spec %d", id, encodedBuffer.size(), request->fd, spec.dest);
}
encodedBuffer.clear();
}
// The dropbox file
if (requests->mainFd() >= 0) {
err = WriteToRequest(id, requests->mainFd(), encodedBuffer, get_default_dropbox_spec());
err = encodedBuffer.strip(get_default_dropbox_spec());
if (err != NO_ERROR) return err; // the buffer data is corrupted.
err = WriteToRequest(id, requests->mainFd(), encodedBuffer);
if (err != NO_ERROR) {
requests->setMainFd(-1);
} else {

3
cmds/incidentd/tests/Reporter_test.cpp
View File

@ -76,8 +76,7 @@ public:
};
protected:
IBinder* onAsBinder() override { return nullptr; };
virtual IBinder* onAsBinder() override { return nullptr; };
};
class ReporterTest : public Test {

130
cmds/incidentd/tests/Section_test.cpp
View File

@ -33,11 +33,27 @@ const string STRING_FIELD_2 = "\x12\vwhatthefuck";
const string FIX64_FIELD_3 = "\x19\xff\xff\xff\xff\xff\xff\xff\xff"; // -1
using namespace android::base;
using namespace android::binder;
using namespace std;
using ::testing::StrEq;
using ::testing::internal::CaptureStdout;
using ::testing::internal::GetCapturedStdout;
class SimpleListener : public IIncidentReportStatusListener
{
public:
SimpleListener() {};
virtual ~SimpleListener() {};
virtual Status onReportStarted() { return Status::ok(); };
virtual Status onReportSectionStatus(int /*section*/, int /*status*/) { return Status::ok(); };
virtual Status onReportFinished() { return Status::ok(); };
virtual Status onReportFailed() { return Status::ok(); };
protected:
virtual IBinder* onAsBinder() override { return nullptr; };
};
// NOTICE: this test requires /system/bin/incident_helper is installed.
TEST(SectionTest, FileSection) {
TemporaryFile tf;
@ -126,4 +142,118 @@ TEST(SectionTest, TestFilterPiiTaggedFields) {
CaptureStdout();
ASSERT_EQ(NO_ERROR, fs.Execute(&requests));
EXPECT_THAT(GetCapturedStdout(), StrEq("\x02\r" + STRING_FIELD_2));
}
TEST(SectionTest, TestBadFdRequest) {
TemporaryFile input;
FileSection fs(NOOP_PARSER, input.path);
ReportRequestSet requests;
ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, input.path, false));
IncidentReportArgs args;
args.setAll(true);
args.setDest(0);
sp<ReportRequest> badFdRequest = new ReportRequest(args, new SimpleListener(), 1234567);
requests.add(badFdRequest);
requests.setMainFd(STDOUT_FILENO);
CaptureStdout();
ASSERT_EQ(NO_ERROR, fs.Execute(&requests));
EXPECT_THAT(GetCapturedStdout(), StrEq("\x02\r" + STRING_FIELD_2));
EXPECT_EQ(badFdRequest->err, -EBADF);
}
TEST(SectionTest, TestBadRequests) {
TemporaryFile input;
FileSection fs(NOOP_PARSER, input.path);
ReportRequestSet requests;
ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, input.path, false));
IncidentReportArgs args;
args.setAll(true);
args.setDest(0);
requests.add(new ReportRequest(args, new SimpleListener(), -1));
EXPECT_EQ(fs.Execute(&requests), -EBADF);
}
TEST(SectionTest, TestMultipleRequests) {
TemporaryFile input, output1, output2, output3;
FileSection fs(NOOP_PARSER, input.path);
ReportRequestSet requests;
ASSERT_TRUE(input.fd != -1);
ASSERT_TRUE(output1.fd != -1);
ASSERT_TRUE(output2.fd != -1);
ASSERT_TRUE(output3.fd != -1);
ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, input.path, false));
IncidentReportArgs args1, args2, args3;
args1.setAll(true);
args1.setDest(0); // LOCAL
args2.setAll(true); // default to explicit
sp<SimpleListener> l = new SimpleListener();
requests.add(new ReportRequest(args1, l, output1.fd));
requests.add(new ReportRequest(args2, l, output2.fd));
requests.add(new ReportRequest(args3, l, output3.fd));
requests.setMainFd(STDOUT_FILENO);
CaptureStdout();
ASSERT_EQ(NO_ERROR, fs.Execute(&requests));
EXPECT_THAT(GetCapturedStdout(), StrEq("\x02\r" + STRING_FIELD_2));
string content, expect;
expect = VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3;
char c = (char) expect.size();
EXPECT_TRUE(ReadFileToString(output1.path, &content));
EXPECT_THAT(content, StrEq(string("\x02") + c + expect));
expect = STRING_FIELD_2 + FIX64_FIELD_3;
c = (char) expect.size();
EXPECT_TRUE(ReadFileToString(output2.path, &content));
EXPECT_THAT(content, StrEq(string("\x02") + c + expect));
// because args3 doesn't set section, so it should receive nothing
EXPECT_TRUE(ReadFileToString(output3.path, &content));
EXPECT_THAT(content, StrEq(""));
}
TEST(SectionTest, TestMultipleRequestsBySpec) {
TemporaryFile input, output1, output2, output3;
FileSection fs(NOOP_PARSER, input.path);
ReportRequestSet requests;
ASSERT_TRUE(input.fd != -1);
ASSERT_TRUE(output1.fd != -1);
ASSERT_TRUE(output2.fd != -1);
ASSERT_TRUE(output3.fd != -1);
ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, input.path, false));
IncidentReportArgs args1, args2, args3, args4;
args1.setAll(true);
args2.setAll(true);
args4.setAll(true);
sp<SimpleListener> l = new SimpleListener();
requests.add(new ReportRequest(args1, l, output1.fd));
requests.add(new ReportRequest(args2, l, output2.fd));
requests.add(new ReportRequest(args3, l, output3.fd));
requests.setMainFd(STDOUT_FILENO);
CaptureStdout();
ASSERT_EQ(NO_ERROR, fs.Execute(&requests));
EXPECT_THAT(GetCapturedStdout(), StrEq("\x02\r" + STRING_FIELD_2));
string content, expect;
expect = STRING_FIELD_2 + FIX64_FIELD_3;
char c = (char) expect.size();
// output1 and output2 are the same
EXPECT_TRUE(ReadFileToString(output1.path, &content));
EXPECT_THAT(content, StrEq(string("\x02") + c + expect));
EXPECT_TRUE(ReadFileToString(output2.path, &content));
EXPECT_THAT(content, StrEq(string("\x02") + c + expect));
// because args3 doesn't set section, so it should receive nothing
EXPECT_TRUE(ReadFileToString(output3.path, &content));
EXPECT_THAT(content, StrEq(""));
}

16
core/java/android/os/IncidentReportArgs.java
View File

@ -35,6 +35,7 @@ public final class IncidentReportArgs implements Parcelable {
private final IntArray mSections = new IntArray();
private final ArrayList<byte[]> mHeaders = new ArrayList<byte[]>();
private boolean mAll;
private int mDest;
/**
* Construct an incident report args with no fields.
@ -69,6 +70,8 @@ public final class IncidentReportArgs implements Parcelable {
for (int i=0; i<N; i++) {
out.writeByteArray(mHeaders.get(i));
}
out.writeInt(mDest);
}
public void readFromParcel(Parcel in) {
@ -85,6 +88,8 @@ public final class IncidentReportArgs implements Parcelable {
for (int i=0; i<N; i++) {
mHeaders.add(in.createByteArray());
}
mDest = in.readInt();
}
public static final Parcelable.Creator<IncidentReportArgs> CREATOR
@ -118,7 +123,8 @@ public final class IncidentReportArgs implements Parcelable {
}
sb.append(", ");
sb.append(mHeaders.size());
sb.append(" headers)");
sb.append(" headers), ");
sb.append("Dest enum value: ").append(mDest);
return sb.toString();
}
@ -132,6 +138,14 @@ public final class IncidentReportArgs implements Parcelable {
}
}
/**
* Set this incident report privacy policy spec.
* @hide
*/
public void setPrivacyPolicy(int dest) {
mDest = dest;
}
/**
* Add this section to the incident report. Skip if the input is smaller than 2 since section
* id are only valid for positive integer as Protobuf field id. Here 1 is reserved for Header.

6
libs/incident/include/android/os/IncidentReportArgs.h
View File

@ -39,12 +39,13 @@ public:
virtual status_t readFromParcel(const Parcel* in);
void setAll(bool all);
void setDest(int dest);
void addSection(int section);
void addHeader(const vector<int8_t>& header);
inline bool all() const { return mAll; };
inline bool all() const { return mAll; }
bool containsSection(int section) const;
inline int dest() const { return mDest; }
inline const set<int>& sections() const { return mSections; }
inline const vector<vector<int8_t>>& headers() const { return mHeaders; }
@ -54,6 +55,7 @@ private:
set<int> mSections;
vector<vector<int8_t>> mHeaders;
bool mAll;
int mDest;
};
}

24
libs/incident/src/IncidentReportArgs.cpp
View File

@ -25,14 +25,16 @@ namespace os {
IncidentReportArgs::IncidentReportArgs()
:mSections(),
mAll(false)
mAll(false),
mDest(-1)
{
}
IncidentReportArgs::IncidentReportArgs(const IncidentReportArgs& that)
:mSections(that.mSections),
mHeaders(that.mHeaders),
mAll(that.mAll)
mAll(that.mAll),
mDest(that.mDest)
{
}
@ -74,6 +76,11 @@ IncidentReportArgs::writeToParcel(Parcel* out) const
}
}
err = out->writeInt32(mDest);
if (err != NO_ERROR) {
return err;
}
return NO_ERROR;
}
@ -120,6 +127,13 @@ IncidentReportArgs::readFromParcel(const Parcel* in)
}
}
int32_t dest;
err = in->readInt32(&dest);
if (err != NO_ERROR) {
return err;
}
mDest = dest;
return OK;
}
@ -132,6 +146,12 @@ IncidentReportArgs::setAll(bool all)
}
}
void
IncidentReportArgs::setDest(int dest)
{
mDest = dest;
}
void
IncidentReportArgs::addSection(int section)
{