Skip to content

Commit

Permalink
[FM] Fix retrieving files if a folder contains thousands of files
Browse files Browse the repository at this point in the history
Tested with 41,000 files in a single folder

Signed-off-by: Muntashir Al-Islam <[email protected]>
  • Loading branch information
MuntashirAkon committed Sep 6, 2023
1 parent 7cbf838 commit 0a22039
Show file tree
Hide file tree
Showing 10 changed files with 120 additions and 105 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

package io.github.muntashirakon.io;

import aosp.android.content.pm.StringParceledListSlice;
import io.github.muntashirakon.io.IOResult;

// Copyright 2022 John "topjohnwu" Wu
Expand All @@ -18,7 +19,7 @@ interface IFileSystemService {
long length(String path);
/* (err, bool) */ IOResult createNewFile(String path);
boolean delete(String path);
String[] list(String path);
StringParceledListSlice list(String path);
boolean mkdir(String path);
boolean mkdirs(String path);
boolean renameTo(String path, String dest);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// SPDX-License-Identifier: Apache-2.0 AND GPL-3.0-or-later
// SPDX-License-Identifier: GPL-3.0-or-later

package aosp.android.content.pm;

Expand All @@ -14,12 +14,10 @@
import java.util.ArrayList;
import java.util.List;

import io.github.muntashirakon.io.IoUtils;

/**
* Transfer a large list of Parcelable objects across an IPC. Splits into
* multiple transactions if needed.
*
* <p>
* Caveat: for efficiency and security, all elements must be the same concrete type.
* In order to avoid writing the class name of each object, we must ensure that
* each object is the same type, or else unparceling then reparceling the data may yield
Expand All @@ -31,7 +29,7 @@ abstract class BaseParceledListSlice<T> implements Parcelable {
private static final String TAG = "ParceledListSlice";
private static final boolean DEBUG = false;

private static final int MAX_IPC_SIZE = IoUtils.DEFAULT_BUFFER_SIZE;
private static final int MAX_IPC_SIZE = 1024 * 50; /*IoUtils.DEFAULT_BUFFER_SIZE*/

private final List<T> mList;

Expand Down Expand Up @@ -69,7 +67,7 @@ public BaseParceledListSlice(List<T> list) {

mList.add(parcelable);

if (DEBUG) Log.d(TAG, "Read inline #" + i + ": " + mList.get(mList.size()-1));
if (DEBUG) Log.d(TAG, "Read inline #" + i + ": " + mList.get(mList.size() - 1));
i++;
}
if (i >= N) {
Expand All @@ -93,7 +91,7 @@ public BaseParceledListSlice(List<T> list) {

mList.add(parcelable);

if (DEBUG) Log.d(TAG, "Read extra #" + i + ": " + mList.get(mList.size()-1));
if (DEBUG) Log.d(TAG, "Read extra #" + i + ": " + mList.get(mList.size() - 1));
i++;
}
reply.recycle();
Expand Down
102 changes: 102 additions & 0 deletions libcore/io/src/main/java/aosp/android/content/pm/ParcelUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
// SPDX-License-Identifier: GPL-3.0-or-later

package aosp.android.content.pm;

import android.os.BadParcelableException;
import android.os.Build;
import android.os.Parcel;
import android.os.Parcelable;
import android.util.Log;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;

import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.HashMap;

public class ParcelUtils {
private static final String TAG = ParcelUtils.class.getSimpleName();

public static <T> void writeParcelableCreator(@NonNull T parcelable, @NonNull Parcel dest) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
dest.writeParcelableCreator((Parcelable) parcelable);
} else {
String name = parcelable.getClass().getName();
dest.writeString(name);
}
}

// Cache of previously looked up CREATOR.createFromParcel() methods for
// particular classes. Keys are the names of the classes, values are
// Method objects.
private static final HashMap<ClassLoader, HashMap<String, Parcelable.Creator<?>>> mCreators = new HashMap<>();

@Nullable
public static Parcelable.Creator<?> readParcelableCreator(@NonNull Parcel from, @Nullable ClassLoader loader) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
return from.readParcelableCreator(loader);
}
String name = from.readString();
if (name == null) {
return null;
}
Parcelable.Creator<?> creator;
synchronized (mCreators) {
HashMap<String, Parcelable.Creator<?>> map = mCreators.get(loader);
if (map == null) {
map = new HashMap<>();
mCreators.put(loader, map);
}
creator = map.get(name);
if (creator == null) {
try {
// If loader == null, explicitly emulate Class.forName(String) "caller
// classloader" behavior.
ClassLoader parcelableClassLoader = (loader == null ? from.getClass().getClassLoader() : loader);
// Avoid initializing the Parcelable class until we know it implements
// Parcelable and has the necessary CREATOR field.
Class<?> parcelableClass = Class.forName(name, false /* initialize */,
parcelableClassLoader);
if (!Parcelable.class.isAssignableFrom(parcelableClass)) {
throw new BadParcelableException("Parcelable protocol requires that the "
+ "class implements Parcelable");
}
Field f = parcelableClass.getField("CREATOR");
if ((f.getModifiers() & Modifier.STATIC) == 0) {
throw new BadParcelableException("Parcelable protocol requires "
+ "the CREATOR object to be static on class " + name);
}
Class<?> creatorType = f.getType();
if (!Parcelable.Creator.class.isAssignableFrom(creatorType)) {
// Fail before calling Field.get(), not after, to avoid initializing
// parcelableClass unnecessarily.
throw new BadParcelableException("Parcelable protocol requires a "
+ "Parcelable.Creator object called "
+ "CREATOR on class " + name);
}
creator = (Parcelable.Creator<?>) f.get(null);
} catch (IllegalAccessException e) {
Log.e(TAG, "Illegal access when unmarshalling: " + name, e);
throw new BadParcelableException(
"IllegalAccessException when unmarshalling: " + name);
} catch (ClassNotFoundException e) {
Log.e(TAG, "Class not found when unmarshalling: " + name, e);
throw new BadParcelableException("ClassNotFoundException when unmarshalling: " + name);
} catch (NoSuchFieldException e) {
throw new BadParcelableException("Parcelable protocol requires a "
+ "Parcelable.Creator object called "
+ "CREATOR on class " + name);
}
if (creator == null) {
throw new BadParcelableException("Parcelable protocol requires a "
+ "non-null Parcelable.Creator object called "
+ "CREATOR on class " + name);
}
map.put(name, creator);
}
}

return creator;
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// SPDX-License-Identifier: Apache-2.0 AND GPL-3.0-or-later
// SPDX-License-Identifier: GPL-3.0-or-later

package aosp.android.content.pm;

Expand All @@ -10,8 +10,6 @@
import java.util.Collections;
import java.util.List;

import io.github.muntashirakon.util.ParcelUtils;

/**
* Transfer a large list of Parcelable objects across an IPC. Splits into
* multiple transactions if needed.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// SPDX-License-Identifier: Apache-2.0 AND GPL-3.0-or-later
// SPDX-License-Identifier: GPL-3.0-or-later

package aosp.android.content.pm;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@

import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import aosp.android.content.pm.StringParceledListSlice;
import io.github.muntashirakon.compat.system.OsCompat;
import io.github.muntashirakon.compat.system.StructTimespec;

Expand Down Expand Up @@ -109,8 +111,9 @@ public boolean delete(String path) {
}

@Override
public String[] list(String path) {
return mCache.get(path).list();
public StringParceledListSlice list(String path) {
String[] list = mCache.get(path).list();
return list != null ? new StringParceledListSlice(Arrays.asList(list)) : null;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
import java.io.FileOutputStream;
import java.io.IOException;

import aosp.android.content.pm.StringParceledListSlice;

// Copyright 2022 John "topjohnwu" Wu
// Copyright 2022 Muntashir Al-Islam
class RemoteFile extends FileImpl<RemoteFile> {
Expand Down Expand Up @@ -313,7 +315,8 @@ public void deleteOnExit() {
@Override
public String[] list() {
try {
return fs.list(getPath());
StringParceledListSlice list = fs.list(getPath());
return list != null ? list.getList().toArray(new String[0]) : null;
} catch (RemoteException e) {
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,16 @@

package io.github.muntashirakon.util;

import android.os.BadParcelableException;
import android.os.Build;
import android.os.Parcel;
import android.os.Parcelable;
import android.util.Log;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.collection.ArraySet;

import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.HashMap;
import java.util.Map;

public class ParcelUtils {
private static final String TAG = ParcelUtils.class.getSimpleName();

/**
* Write an array set to the parcel.
*
Expand Down Expand Up @@ -70,86 +62,4 @@ public static <K, V> Map<K, V> readMap(@NonNull Parcel parcel, @Nullable ClassLo
}
return map;
}

public static <T> void writeParcelableCreator(@NonNull T parcelable, @NonNull Parcel dest) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
dest.writeParcelableCreator((Parcelable) parcelable);
} else {
String name = parcelable.getClass().getName();
dest.writeString(name);
}
}

// Cache of previously looked up CREATOR.createFromParcel() methods for
// particular classes. Keys are the names of the classes, values are
// Method objects.
private static final HashMap<ClassLoader, HashMap<String, Parcelable.Creator<?>>> mCreators = new HashMap<>();

@Nullable
public static Parcelable.Creator<?> readParcelableCreator(@NonNull Parcel from, @Nullable ClassLoader loader) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
return from.readParcelableCreator(loader);
}
String name = from.readString();
if (name == null) {
return null;
}
Parcelable.Creator<?> creator;
synchronized (mCreators) {
HashMap<String, Parcelable.Creator<?>> map = mCreators.get(loader);
if (map == null) {
map = new HashMap<>();
mCreators.put(loader, map);
}
creator = map.get(name);
if (creator == null) {
try {
// If loader == null, explicitly emulate Class.forName(String) "caller
// classloader" behavior.
ClassLoader parcelableClassLoader = (loader == null ? from.getClass().getClassLoader() : loader);
// Avoid initializing the Parcelable class until we know it implements
// Parcelable and has the necessary CREATOR field.
Class<?> parcelableClass = Class.forName(name, false /* initialize */,
parcelableClassLoader);
if (!Parcelable.class.isAssignableFrom(parcelableClass)) {
throw new BadParcelableException("Parcelable protocol requires that the "
+ "class implements Parcelable");
}
Field f = parcelableClass.getField("CREATOR");
if ((f.getModifiers() & Modifier.STATIC) == 0) {
throw new BadParcelableException("Parcelable protocol requires "
+ "the CREATOR object to be static on class " + name);
}
Class<?> creatorType = f.getType();
if (!Parcelable.Creator.class.isAssignableFrom(creatorType)) {
// Fail before calling Field.get(), not after, to avoid initializing
// parcelableClass unnecessarily.
throw new BadParcelableException("Parcelable protocol requires a "
+ "Parcelable.Creator object called "
+ "CREATOR on class " + name);
}
creator = (Parcelable.Creator<?>) f.get(null);
} catch (IllegalAccessException e) {
Log.e(TAG, "Illegal access when unmarshalling: " + name, e);
throw new BadParcelableException(
"IllegalAccessException when unmarshalling: " + name);
} catch (ClassNotFoundException e) {
Log.e(TAG, "Class not found when unmarshalling: " + name, e);
throw new BadParcelableException("ClassNotFoundException when unmarshalling: " + name);
} catch (NoSuchFieldException e) {
throw new BadParcelableException("Parcelable protocol requires a "
+ "Parcelable.Creator object called "
+ "CREATOR on class " + name);
}
if (creator == null) {
throw new BadParcelableException("Parcelable protocol requires a "
+ "non-null Parcelable.Creator object called "
+ "CREATOR on class " + name);
}
map.put(name, creator);
}
}

return creator;
}
}

0 comments on commit 0a22039

Please sign in to comment.