am bae6fe24: Merge "Support streaming of compressed assets > 1 megabyte" into gingerbread

Merge commit 'bae6fe242ff9a97840ff0c5d9d87c0575d261682' into gingerbread-plus-aosp

* commit 'bae6fe242ff9a97840ff0c5d9d87c0575d261682':
  Support streaming of compressed assets > 1 megabyte
This commit is contained in:
Christopher Tate
2010-07-28 15:38:38 -07:00
committed by Android Git Automerger
11 changed files with 574 additions and 40 deletions

View File

@ -61,15 +61,6 @@ public:
ACCESS_BUFFER,
} AccessMode;
enum {
/* data larger than this does not get uncompressed into a buffer */
#ifdef HAVE_ANDROID_OS
UNCOMPRESS_DATA_MAX = 1 * 1024 * 1024
#else
UNCOMPRESS_DATA_MAX = 2 * 1024 * 1024
#endif
};
/*
* Read data from the current offset. Returns the actual number of
* bytes read, 0 on EOF, or -1 on error.
@ -317,6 +308,8 @@ private:
FileMap* mMap; // for memory-mapped input
int mFd; // for file input
class StreamingZipInflater* mZipInflater; // for streaming large compressed assets
unsigned char* mBuf; // for getBuffer()
};

View File

@ -0,0 +1,82 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __LIBS_STREAMINGZIPINFLATER_H
#define __LIBS_STREAMINGZIPINFLATER_H
#include <unistd.h>
#include <inttypes.h>
#include <zlib.h>
namespace android {
class StreamingZipInflater {
public:
static const size_t INPUT_CHUNK_SIZE = 64 * 1024;
static const size_t OUTPUT_CHUNK_SIZE = 64 * 1024;
// Flavor that pages in the compressed data from a fd
StreamingZipInflater(int fd, off_t compDataStart, size_t uncompSize, size_t compSize);
// Flavor that gets the compressed data from an in-memory buffer
StreamingZipInflater(class FileMap* dataMap, size_t uncompSize);
~StreamingZipInflater();
// read 'count' bytes of uncompressed data from the current position. outBuf may
// be NULL, in which case the data is consumed and discarded.
ssize_t read(void* outBuf, size_t count);
// seeking backwards requires uncompressing fom the beginning, so is very
// expensive. seeking forwards only requires uncompressing from the current
// position to the destination.
off_t seekAbsolute(off_t absoluteInputPosition);
private:
void initInflateState();
int readNextChunk();
// where to find the uncompressed data
int mFd;
off_t mInFileStart; // where the compressed data lives in the file
class FileMap* mDataMap;
z_stream mInflateState;
bool mStreamNeedsInit;
// output invariants for this asset
uint8_t* mOutBuf; // output buf for decompressed bytes
size_t mOutBufSize; // allocated size of mOutBuf
size_t mOutTotalSize; // total uncompressed size of the blob
// current output state bookkeeping
off_t mOutCurPosition; // current position in total offset
size_t mOutLastDecoded; // last decoded byte + 1 in mOutbuf
size_t mOutDeliverable; // next undelivered byte of decoded output in mOutBuf
// input invariants
uint8_t* mInBuf;
size_t mInBufSize; // allocated size of mInBuf;
size_t mInTotalSize; // total size of compressed data for this blob
// input state bookkeeping
size_t mInNextChunkOffset; // offset from start of blob at which the next input chunk lies
// the z_stream contains state about input block consumption
};
}
#endif

View File

@ -33,6 +33,7 @@ commonSources:= \
SharedBuffer.cpp \
Static.cpp \
StopWatch.cpp \
StreamingZipInflater.cpp \
String8.cpp \
String16.cpp \
StringArray.cpp \

View File

@ -24,6 +24,7 @@
#include <utils/Asset.h>
#include <utils/Atomic.h>
#include <utils/FileMap.h>
#include <utils/StreamingZipInflater.h>
#include <utils/ZipUtils.h>
#include <utils/ZipFileRO.h>
#include <utils/Log.h>
@ -659,7 +660,7 @@ const void* _FileAsset::ensureAlignment(FileMap* map)
*/
_CompressedAsset::_CompressedAsset(void)
: mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0),
mMap(NULL), mFd(-1), mBuf(NULL)
mMap(NULL), mFd(-1), mZipInflater(NULL), mBuf(NULL)
{
}
@ -698,6 +699,10 @@ status_t _CompressedAsset::openChunk(int fd, off_t offset,
mFd = fd;
assert(mBuf == NULL);
if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
mZipInflater = new StreamingZipInflater(mFd, offset, uncompressedLen, compressedLen);
}
return NO_ERROR;
}
@ -724,6 +729,9 @@ status_t _CompressedAsset::openChunk(FileMap* dataMap, int compressionMethod,
mUncompressedLen = uncompressedLen;
assert(mOffset == 0);
if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
mZipInflater = new StreamingZipInflater(dataMap, uncompressedLen);
}
return NO_ERROR;
}
@ -739,26 +747,29 @@ ssize_t _CompressedAsset::read(void* buf, size_t count)
assert(mOffset >= 0 && mOffset <= mUncompressedLen);
// TODO: if mAccessMode == ACCESS_STREAMING, use zlib more cleverly
/* If we're relying on a streaming inflater, go through that */
if (mZipInflater) {
actual = mZipInflater->read(buf, count);
} else {
if (mBuf == NULL) {
if (getBuffer(false) == NULL)
return -1;
}
assert(mBuf != NULL);
if (mBuf == NULL) {
if (getBuffer(false) == NULL)
return -1;
/* adjust count if we're near EOF */
maxLen = mUncompressedLen - mOffset;
if (count > maxLen)
count = maxLen;
if (!count)
return 0;
/* copy from buffer */
//printf("comp buf read\n");
memcpy(buf, (char*)mBuf + mOffset, count);
actual = count;
}
assert(mBuf != NULL);
/* adjust count if we're near EOF */
maxLen = mUncompressedLen - mOffset;
if (count > maxLen)
count = maxLen;
if (!count)
return 0;
/* copy from buffer */
//printf("comp buf read\n");
memcpy(buf, (char*)mBuf + mOffset, count);
actual = count;
mOffset += actual;
return actual;
@ -780,6 +791,9 @@ off_t _CompressedAsset::seek(off_t offset, int whence)
if (newPosn == (off_t) -1)
return newPosn;
if (mZipInflater) {
mZipInflater->seekAbsolute(newPosn);
}
mOffset = newPosn;
return mOffset;
}
@ -793,10 +807,12 @@ void _CompressedAsset::close(void)
mMap->release();
mMap = NULL;
}
if (mBuf != NULL) {
delete[] mBuf;
mBuf = NULL;
}
delete[] mBuf;
mBuf = NULL;
delete mZipInflater;
mZipInflater = NULL;
if (mFd > 0) {
::close(mFd);
@ -817,12 +833,6 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
if (mBuf != NULL)
return mBuf;
if (mUncompressedLen > UNCOMPRESS_DATA_MAX) {
LOGD("Data exceeds UNCOMPRESS_DATA_MAX (%ld vs %d)\n",
(long) mUncompressedLen, UNCOMPRESS_DATA_MAX);
goto bail;
}
/*
* Allocate a buffer and read the file into it.
*/
@ -853,7 +863,13 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
goto bail;
}
/* success! */
/*
* Success - now that we have the full asset in RAM we
* no longer need the streaming inflater
*/
delete mZipInflater;
mZipInflater = NULL;
mBuf = buf;
buf = NULL;

View File

@ -0,0 +1,224 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define LOG_TAG "szipinf"
#include <utils/Log.h>
#include <utils/FileMap.h>
#include <utils/StreamingZipInflater.h>
#include <string.h>
#include <assert.h>
static inline size_t min(size_t a, size_t b) { return (a < b) ? a : b; }
using namespace android;
/*
* Streaming access to compressed asset data in an open fd
*/
StreamingZipInflater::StreamingZipInflater(int fd, off_t compDataStart,
size_t uncompSize, size_t compSize) {
mFd = fd;
mDataMap = NULL;
mInFileStart = compDataStart;
mOutTotalSize = uncompSize;
mInTotalSize = compSize;
mInBufSize = StreamingZipInflater::INPUT_CHUNK_SIZE;
mInBuf = new uint8_t[mInBufSize];
mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
mOutBuf = new uint8_t[mOutBufSize];
initInflateState();
}
/*
* Streaming access to compressed data held in an mmapped region of memory
*/
StreamingZipInflater::StreamingZipInflater(FileMap* dataMap, size_t uncompSize) {
mFd = -1;
mDataMap = dataMap;
mOutTotalSize = uncompSize;
mInTotalSize = dataMap->getDataLength();
mInBuf = (uint8_t*) dataMap->getDataPtr();
mInBufSize = mInTotalSize;
mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
mOutBuf = new uint8_t[mOutBufSize];
initInflateState();
}
StreamingZipInflater::~StreamingZipInflater() {
// tear down the in-flight zip state just in case
::inflateEnd(&mInflateState);
if (mDataMap == NULL) {
delete [] mInBuf;
}
delete [] mOutBuf;
}
void StreamingZipInflater::initInflateState() {
LOGD("Initializing inflate state");
memset(&mInflateState, 0, sizeof(mInflateState));
mInflateState.zalloc = Z_NULL;
mInflateState.zfree = Z_NULL;
mInflateState.opaque = Z_NULL;
mInflateState.next_in = (Bytef*)mInBuf;
mInflateState.next_out = (Bytef*) mOutBuf;
mInflateState.avail_out = mOutBufSize;
mInflateState.data_type = Z_UNKNOWN;
mOutLastDecoded = mOutDeliverable = mOutCurPosition = 0;
mInNextChunkOffset = 0;
mStreamNeedsInit = true;
if (mDataMap == NULL) {
::lseek(mFd, mInFileStart, SEEK_SET);
mInflateState.avail_in = 0; // set when a chunk is read in
} else {
mInflateState.avail_in = mInBufSize;
}
}
/*
* Basic approach:
*
* 1. If we have undelivered uncompressed data, send it. At this point
* either we've satisfied the request, or we've exhausted the available
* output data in mOutBuf.
*
* 2. While we haven't sent enough data to satisfy the request:
* 0. if the request is for more data than exists, bail.
* a. if there is no input data to decode, read some into the input buffer
* and readjust the z_stream input pointers
* b. point the output to the start of the output buffer and decode what we can
* c. deliver whatever output data we can
*/
ssize_t StreamingZipInflater::read(void* outBuf, size_t count) {
uint8_t* dest = (uint8_t*) outBuf;
size_t bytesRead = 0;
size_t toRead = min(count, size_t(mOutTotalSize - mOutCurPosition));
while (toRead > 0) {
// First, write from whatever we already have decoded and ready to go
size_t deliverable = min(toRead, mOutLastDecoded - mOutDeliverable);
if (deliverable > 0) {
if (outBuf != NULL) memcpy(dest, mOutBuf + mOutDeliverable, deliverable);
mOutDeliverable += deliverable;
mOutCurPosition += deliverable;
dest += deliverable;
bytesRead += deliverable;
toRead -= deliverable;
}
// need more data? time to decode some.
if (toRead > 0) {
// if we don't have any data to decode, read some in. If we're working
// from mmapped data this won't happen, because the clipping to total size
// will prevent reading off the end of the mapped input chunk.
if (mInflateState.avail_in == 0) {
int err = readNextChunk();
if (err < 0) {
LOGE("Unable to access asset data: %d", err);
if (!mStreamNeedsInit) {
::inflateEnd(&mInflateState);
initInflateState();
}
return -1;
}
}
// we know we've drained whatever is in the out buffer now, so just
// start from scratch there, reading all the input we have at present.
mInflateState.next_out = (Bytef*) mOutBuf;
mInflateState.avail_out = mOutBufSize;
/*
LOGD("Inflating to outbuf: avail_in=%u avail_out=%u next_in=%p next_out=%p",
mInflateState.avail_in, mInflateState.avail_out,
mInflateState.next_in, mInflateState.next_out);
*/
int result = Z_OK;
if (mStreamNeedsInit) {
LOGI("Initializing zlib to inflate");
result = inflateInit2(&mInflateState, -MAX_WBITS);
mStreamNeedsInit = false;
}
if (result == Z_OK) result = ::inflate(&mInflateState, Z_SYNC_FLUSH);
if (result < 0) {
// Whoops, inflation failed
LOGE("Error inflating asset: %d", result);
::inflateEnd(&mInflateState);
initInflateState();
return -1;
} else {
if (result == Z_STREAM_END) {
// we know we have to have reached the target size here and will
// not try to read any further, so just wind things up.
::inflateEnd(&mInflateState);
}
// Note how much data we got, and off we go
mOutDeliverable = 0;
mOutLastDecoded = mOutBufSize - mInflateState.avail_out;
}
}
}
return bytesRead;
}
int StreamingZipInflater::readNextChunk() {
assert(mDataMap == NULL);
if (mInNextChunkOffset < mInTotalSize) {
size_t toRead = min(mInBufSize, mInTotalSize - mInNextChunkOffset);
if (toRead > 0) {
ssize_t didRead = ::read(mFd, mInBuf, toRead);
//LOGD("Reading input chunk, size %08x didread %08x", toRead, didRead);
if (didRead < 0) {
// TODO: error
LOGE("Error reading asset data");
return didRead;
} else {
mInNextChunkOffset += didRead;
mInflateState.next_in = (Bytef*) mInBuf;
mInflateState.avail_in = didRead;
}
}
}
return 0;
}
// seeking backwards requires uncompressing fom the beginning, so is very
// expensive. seeking forwards only requires uncompressing from the current
// position to the destination.
off_t StreamingZipInflater::seekAbsolute(off_t absoluteInputPosition) {
if (absoluteInputPosition < mOutCurPosition) {
// rewind and reprocess the data from the beginning
if (!mStreamNeedsInit) {
::inflateEnd(&mInflateState);
}
initInflateState();
read(NULL, absoluteInputPosition);
} else if (absoluteInputPosition > mOutCurPosition) {
read(NULL, absoluteInputPosition - mOutCurPosition);
}
// else if the target position *is* our current position, do nothing
return absoluteInputPosition;
}

View File

@ -0,0 +1,11 @@
LOCAL_PATH:= $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE_TAGS := optional
LOCAL_SRC_FILES := $(call all-subdir-java-files)
LOCAL_PACKAGE_NAME := LargeAssetTest
LOCAL_CERTIFICATE := platform
include $(BUILD_PACKAGE)

View File

@ -0,0 +1,28 @@
<!-- Copyright (C) 2010 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.android.largeassettest">
<application>
<activity android:name="LargeAssetTest" android:label="Large Asset Test">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.DEFAULT" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
</application>
</manifest>

Binary file not shown.

View File

@ -0,0 +1,52 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2010 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:orientation="vertical"
>
<TextView
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="25dp"
android:textColor="#ffffffff"
android:text="@string/prompt"
/>
<TextView android:id="@+id/result"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_weight="1"
android:layout_marginTop="25dp"
android:textSize="24sp"
android:textColor="#ffffffff"
/>
<Button android:id="@+id/validate"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/button_text" />
<TextView
android:layout_height="wrap_content"
android:layout_width="match_parent"
android:layout_weight="1"
/>
</LinearLayout>

View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2010 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources>
<string name="prompt">Click the button below to read and validate the large binary asset.</string>
<string name="button_text">Validate the asset</string>
</resources>

View File

@ -0,0 +1,104 @@
/*
* Copyright (C) 2006 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.largeassettest;
import android.app.Activity;
import android.content.Context;
import android.content.res.AssetManager;
import android.os.AsyncTask;
import android.os.Bundle;
import android.util.Log;
import android.view.View;
import android.widget.Button;
import android.widget.TextView;
import java.io.InputStream;
import java.io.IOException;
/**
* Skeleton to test large-asset handling. The asset in question is one million
* four-byte integers, in ascending numeric order.
*/
public class LargeAssetTest extends Activity {
Button mValidateButton;
TextView mResultText;
Validator mValidateThread;
@Override
protected void onCreate(Bundle icicle) {
super.onCreate(icicle);
setContentView(R.layout.lat);
mResultText = (TextView) findViewById(R.id.result);
mValidateButton = (Button) findViewById(R.id.validate);
mValidateButton.setOnClickListener(mClickListener);
}
View.OnClickListener mClickListener = new View.OnClickListener() {
public void onClick(View v) {
mValidateButton.setEnabled(false);
mValidateThread = new Validator();
mValidateThread.execute(LargeAssetTest.this.getAssets());
}
};
/**
* Validation happens in a separate thread
*/
class Validator extends AsyncTask<AssetManager, Integer, Boolean> {
static final String TAG = "Validator";
@Override
protected Boolean doInBackground(AssetManager... params) {
AssetManager am = params[0];
try {
InputStream is = am.open("million-ints", AssetManager.ACCESS_STREAMING);
byte[] buf = new byte[4];
for (int i = 0; i < 1000000; i++) {
int num = is.read(buf, 0, 4);
if (num != 4) {
Log.e(TAG, "Wanted 4 bytes but read " + num);
return false;
}
// the byte array is stored in the asset in little-endian order
int value = (buf[3] << 24) + ((buf[2] & 0xFF) << 16)
+ ((buf[1] & 0xFF) << 8) + (buf[0] & 0xFF);
if (value != i) {
Log.e(TAG, "Mismatch: index " + i + " : value " + value);
return false;
}
}
is.close();
} catch (IOException e) {
Log.w(TAG, "Couldn't open asset", e);
return false;
}
Log.i(TAG, "Finished, reporting valid");
return true;
}
@Override
protected void onPostExecute(Boolean result) {
CharSequence text = (result) ? "Valid!" : "NOT VALID";
mResultText.setText(text);
mValidateButton.setEnabled(true);
}
}
}