From bb699e41c90789f3cea059faef98c846915b0d07 Mon Sep 17 00:00:00 2001 From: tofunmi Date: Fri, 28 Jul 2023 19:00:35 +0100 Subject: [PATCH] Add png extractor Implementing a basic extractor that reads the whole png file into the trackOutput as one sample. PiperOrigin-RevId: 551897619 --- RELEASENOTES.md | 2 + .../androidx/media3/common/MimeTypes.java | 1 + .../media3/extractor/ImageExtractorUtil.java | 32 +++++ .../media3/extractor/jpeg/JpegExtractor.java | 8 +- .../media3/extractor/png/PngExtractor.java | 129 ++++++++++++++++++ .../media3/extractor/png/package-info.java | 19 +++ .../extractor/png/PngExtractorTest.java | 41 ++++++ .../png/non-motion-photo-shortened.png.0.dump | 11 ++ ...on-photo-shortened.png.unknown_length.dump | 11 ++ 9 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 libraries/extractor/src/main/java/androidx/media3/extractor/ImageExtractorUtil.java create mode 100644 libraries/extractor/src/main/java/androidx/media3/extractor/png/PngExtractor.java create mode 100644 libraries/extractor/src/main/java/androidx/media3/extractor/png/package-info.java create mode 100644 libraries/extractor/src/test/java/androidx/media3/extractor/png/PngExtractorTest.java create mode 100644 libraries/test_data/src/test/assets/extractordumps/png/non-motion-photo-shortened.png.0.dump create mode 100644 libraries/test_data/src/test/assets/extractordumps/png/non-motion-photo-shortened.png.unknown_length.dump diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 317a7128db..be32d0f57d 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -40,6 +40,8 @@ object duration (d) and measured throughput (mtp). * Rename `MimeTypes.TEXT_EXOPLAYER_CUES` to `MimeTypes.APPLICATION_MEDIA3_CUES`. + * Add `PngExtractor` that sends and reads a whole png file into the the + `TrackOutput` as one sample. * Transformer: * Parse EXIF rotation data for image inputs. * Remove `TransformationRequest.HdrMode` annotation type and its diff --git a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java index 90fb3c5dc9..b1d60d9c79 100644 --- a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java +++ b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java @@ -155,6 +155,7 @@ public final class MimeTypes { // image/ MIME types public static final String IMAGE_JPEG = BASE_TYPE_IMAGE + "/jpeg"; + @UnstableApi public static final String IMAGE_PNG = BASE_TYPE_IMAGE + "/png"; /** * A non-standard codec string for E-AC3-JOC. Use of this constant allows for disambiguation diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ImageExtractorUtil.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ImageExtractorUtil.java new file mode 100644 index 0000000000..744820956d --- /dev/null +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ImageExtractorUtil.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.extractor; + +import androidx.media3.common.util.UnstableApi; +import androidx.media3.extractor.mp4.Mp4Extractor; + +/** Utilities for image extractors. */ +@UnstableApi +public class ImageExtractorUtil { + + /** + * The identifier to use for the image track. Chosen to avoid colliding with track IDs used by + * {@link Mp4Extractor} for motion photos. + */ + public static final int IMAGE_TRACK_ID = 1024; + + private ImageExtractorUtil() {} +} diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegExtractor.java index 104956a216..f9fc1d98b9 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegExtractor.java @@ -16,6 +16,7 @@ package androidx.media3.extractor.jpeg; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.extractor.ImageExtractorUtil.IMAGE_TRACK_ID; import static java.lang.annotation.ElementType.TYPE_USE; import androidx.annotation.IntDef; @@ -74,12 +75,6 @@ public final class JpegExtractor implements Extractor { private static final int MARKER_APP1 = 0xFFE1; // Application data 1 marker private static final String HEADER_XMP_APP1 = "http://ns.adobe.com/xap/1.0/"; - /** - * The identifier to use for the image track. Chosen to avoid colliding with track IDs used by - * {@link Mp4Extractor} for motion photos. - */ - private static final int IMAGE_TRACK_ID = 1024; - private final ParsableByteArray scratch; private @MonotonicNonNull ExtractorOutput extractorOutput; @@ -278,6 +273,7 @@ public final class JpegExtractor implements Extractor { private void outputImageTrack(Metadata.Entry... metadataEntries) { TrackOutput imageTrackOutput = checkNotNull(extractorOutput).track(IMAGE_TRACK_ID, C.TRACK_TYPE_IMAGE); + // TODO(b/289989902): Set the rotationDegrees in format so images can be decoded correctly. imageTrackOutput.format( new Format.Builder() .setContainerMimeType(MimeTypes.IMAGE_JPEG) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/png/PngExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/png/PngExtractor.java new file mode 100644 index 0000000000..8429bd807f --- /dev/null +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/png/PngExtractor.java @@ -0,0 +1,129 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.extractor.png; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.extractor.ImageExtractorUtil.IMAGE_TRACK_ID; +import static java.lang.annotation.ElementType.TYPE_USE; + +import androidx.annotation.IntDef; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.extractor.Extractor; +import androidx.media3.extractor.ExtractorInput; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.PositionHolder; +import androidx.media3.extractor.SeekMap; +import androidx.media3.extractor.TrackOutput; +import java.io.IOException; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Extracts data from the PNG container format. */ +@UnstableApi +// TODO: b/289989902 - Move methods of this class into ImageExtractorUtil once there are multiple +// image extractors. +public class PngExtractor implements Extractor { + + /** Parser states. */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @IntDef({STATE_READING_IMAGE, STATE_ENDED}) + private @interface State {} + + private static final int STATE_READING_IMAGE = 1; + private static final int STATE_ENDED = 2; + + private static final int PNG_FILE_SIGNATURE_LENGTH = 2; + // See PNG (Portable Network Graphics) Specification, Version 1.2, Section 12.12 and Section 3.1. + private static final int PNG_FILE_SIGNATURE = 0x8950; + private static final int FIXED_READ_LENGTH = 1024; + + private final ParsableByteArray scratch; + + private @State int state; + private @MonotonicNonNull ExtractorOutput extractorOutput; + + /** Creates an instance. */ + public PngExtractor() { + scratch = new ParsableByteArray(PNG_FILE_SIGNATURE_LENGTH); + } + + @Override + public boolean sniff(ExtractorInput input) throws IOException { + scratch.reset(/* limit= */ PNG_FILE_SIGNATURE_LENGTH); + input.peekFully(scratch.getData(), /* offset= */ 0, PNG_FILE_SIGNATURE_LENGTH); + return scratch.readUnsignedShort() == PNG_FILE_SIGNATURE; + } + + @Override + public void init(ExtractorOutput output) { + extractorOutput = output; + outputImageTrackAndSeekMap(); + } + + @Override + public @ReadResult int read(ExtractorInput input, PositionHolder seekPosition) + throws IOException { + switch (state) { + case STATE_READING_IMAGE: + readSegment(input); + return RESULT_CONTINUE; + case STATE_ENDED: + return RESULT_END_OF_INPUT; + default: + throw new IllegalStateException(); + } + } + + private void readSegment(ExtractorInput input) throws IOException { + TrackOutput trackOutput = + checkNotNull(extractorOutput).track(IMAGE_TRACK_ID, C.TRACK_TYPE_IMAGE); + int result = trackOutput.sampleData(input, FIXED_READ_LENGTH, /* allowEndOfInput= */ true); + if (result == C.RESULT_END_OF_INPUT) { + state = STATE_ENDED; + } + } + + private void outputImageTrackAndSeekMap() { + ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput); + TrackOutput imageTrackOutput = extractorOutput.track(IMAGE_TRACK_ID, C.TRACK_TYPE_IMAGE); + imageTrackOutput.format(new Format.Builder().setContainerMimeType(MimeTypes.IMAGE_PNG).build()); + extractorOutput.endTracks(); + extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET)); + state = STATE_READING_IMAGE; + } + + @Override + public void seek(long position, long timeUs) { + if (position == 0) { + state = STATE_READING_IMAGE; + } + } + + @Override + public void release() { + // Do nothing. + + } +} diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/png/package-info.java b/libraries/extractor/src/main/java/androidx/media3/extractor/png/package-info.java new file mode 100644 index 0000000000..4f208aa03e --- /dev/null +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/png/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +package androidx.media3.extractor.png; + +import androidx.media3.common.util.NonNullApi; diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/png/PngExtractorTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/png/PngExtractorTest.java new file mode 100644 index 0000000000..60e052b3f8 --- /dev/null +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/png/PngExtractorTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.extractor.png; + +import androidx.media3.test.utils.ExtractorAsserts; +import com.google.common.collect.ImmutableList; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.ParameterizedRobolectricTestRunner; + +/** Unit tests for {@link PngExtractor}. */ +@RunWith(ParameterizedRobolectricTestRunner.class) +public final class PngExtractorTest { + + @ParameterizedRobolectricTestRunner.Parameters(name = "{0}") + public static ImmutableList params() { + return ExtractorAsserts.configs(); + } + + @ParameterizedRobolectricTestRunner.Parameter + public ExtractorAsserts.SimulationConfig simulationConfig; + + @Test + public void samplePng() throws Exception { + ExtractorAsserts.assertBehavior( + PngExtractor::new, "media/png/non-motion-photo-shortened.png", simulationConfig); + } +} diff --git a/libraries/test_data/src/test/assets/extractordumps/png/non-motion-photo-shortened.png.0.dump b/libraries/test_data/src/test/assets/extractordumps/png/non-motion-photo-shortened.png.0.dump new file mode 100644 index 0000000000..59edcbfd97 --- /dev/null +++ b/libraries/test_data/src/test/assets/extractordumps/png/non-motion-photo-shortened.png.0.dump @@ -0,0 +1,11 @@ +seekMap: + isSeekable = false + duration = UNSET TIME + getPosition(0) = [[timeUs=0, position=0]] +numberOfTracks = 1 +track 1024: + total output bytes = 29063 + sample count = 0 + format 0: + containerMimeType = image/png +tracksEnded = true diff --git a/libraries/test_data/src/test/assets/extractordumps/png/non-motion-photo-shortened.png.unknown_length.dump b/libraries/test_data/src/test/assets/extractordumps/png/non-motion-photo-shortened.png.unknown_length.dump new file mode 100644 index 0000000000..59edcbfd97 --- /dev/null +++ b/libraries/test_data/src/test/assets/extractordumps/png/non-motion-photo-shortened.png.unknown_length.dump @@ -0,0 +1,11 @@ +seekMap: + isSeekable = false + duration = UNSET TIME + getPosition(0) = [[timeUs=0, position=0]] +numberOfTracks = 1 +track 1024: + total output bytes = 29063 + sample count = 0 + format 0: + containerMimeType = image/png +tracksEnded = true