pxl8/src/pxl8_ase.c

#include <stdlib.h>
#include <string.h>

#define MINIZ_NO_STDIO
#define MINIZ_NO_TIME
#define MINIZ_NO_ARCHIVE_APIS
#define MINIZ_NO_ARCHIVE_WRITING_APIS
#define MINIZ_NO_DEFLATE_APIS

#include <miniz.h>

#include "pxl8_ase.h"
#include "pxl8_io.h"
#include "pxl8_macros.h"

static pxl8_result parse_ase_header(pxl8_stream* stream, pxl8_ase_header* header) {
    header->file_size = pxl8_read_u32(stream);
    header->magic = pxl8_read_u16(stream);
    header->frames = pxl8_read_u16(stream);
    header->width = pxl8_read_u16(stream);
    header->height = pxl8_read_u16(stream);
    header->color_depth = pxl8_read_u16(stream);
    header->flags = pxl8_read_u32(stream);
    header->speed = pxl8_read_u16(stream);
    pxl8_skip_bytes(stream, 4);
    header->transparent_index = pxl8_read_u32(stream);
    header->n_colors = pxl8_read_u8(stream);
    header->pixel_width = pxl8_read_u8(stream);
    header->pixel_height = pxl8_read_u8(stream);
    header->grid_x = pxl8_read_i16(stream);
    header->grid_y = pxl8_read_i16(stream);
    header->grid_width = pxl8_read_u16(stream);
    header->grid_height = pxl8_read_u16(stream);

    if (header->magic != PXL8_ASE_MAGIC) {
        pxl8_error("Invalid ASE file magic: 0x%04X", header->magic);
        return PXL8_ERROR_ASE_INVALID_MAGIC;
    }

    return PXL8_OK;
}

static pxl8_result parse_old_palette_chunk(pxl8_stream* stream, pxl8_ase_palette* palette) {
    u16 packet_count = pxl8_read_u16(stream);

    u32 total_colors = 0;
    u32 temp_pos = pxl8_stream_position(stream);
    for (u16 packet = 0; packet < packet_count; packet++) {
        u8 skip_colors = pxl8_read_u8(stream);
        u8 colors_in_packet = pxl8_read_u8(stream);
        u32 actual_colors = (colors_in_packet == 0) ? 256 : colors_in_packet;
        total_colors += skip_colors + actual_colors;
        pxl8_skip_bytes(stream, actual_colors * 3);
    }

    palette->entry_count = total_colors;
    palette->first_color = 0;
    palette->last_color = total_colors - 1;
    palette->colors = (u32*)malloc(total_colors * sizeof(u32));
    if (!palette->colors) {
        return PXL8_ERROR_OUT_OF_MEMORY;
    }

    pxl8_stream_seek(stream, temp_pos);

    u32 color_index = 0;
    for (u16 packet = 0; packet < packet_count; packet++) {
        u8 skip_colors = pxl8_read_u8(stream);
        u8 colors_in_packet = pxl8_read_u8(stream);
        u32 actual_colors = (colors_in_packet == 0) ? 256 : colors_in_packet;

        for (u32 skip = 0; skip < skip_colors && color_index < total_colors; skip++) {
            palette->colors[color_index++] = 0xFF000000;
        }

        for (u32 color = 0; color < actual_colors && color_index < total_colors; color++) {
            u8 r = pxl8_read_u8(stream);
            u8 g = pxl8_read_u8(stream);
            u8 b = pxl8_read_u8(stream);
            palette->colors[color_index++] = 0xFF000000 | (b << 16) | (g << 8) | r;
        }
    }

    return PXL8_OK;
}

static pxl8_result parse_layer_chunk(pxl8_stream* stream, pxl8_ase_layer* layer) {
    layer->flags = pxl8_read_u16(stream);
    layer->layer_type = pxl8_read_u16(stream);
    layer->child_level = pxl8_read_u16(stream);
    pxl8_skip_bytes(stream, 4);
    layer->blend_mode = pxl8_read_u16(stream);
    layer->opacity = pxl8_read_u8(stream);
    pxl8_skip_bytes(stream, 3);

    u16 name_len = pxl8_read_u16(stream);
    if (name_len > 0) {
        layer->name = (char*)malloc(name_len + 1);
        if (!layer->name) return PXL8_ERROR_OUT_OF_MEMORY;
        pxl8_read_bytes(stream, layer->name, name_len);
        layer->name[name_len] = '\0';
    } else {
        layer->name = NULL;
    }

    return PXL8_OK;
}

static pxl8_result parse_palette_chunk(pxl8_stream* stream, pxl8_ase_palette* palette) {
    pxl8_skip_bytes(stream, 4);
    palette->first_color = pxl8_read_u32(stream);
    palette->last_color = pxl8_read_u32(stream);
    pxl8_skip_bytes(stream, 8);

    u32 color_count = palette->last_color - palette->first_color + 1;

    if (color_count == 0 || color_count > 65536) {
        return PXL8_ERROR_ASE_MALFORMED_CHUNK;
    }

    palette->colors = (u32*)malloc(color_count * sizeof(u32));
    if (!palette->colors) {
        return PXL8_ERROR_OUT_OF_MEMORY;
    }

    palette->entry_count = color_count;

    for (u32 i = 0; i < color_count; i++) {
        u16 flags = pxl8_read_u16(stream);
        u8 r = pxl8_read_u8(stream);
        u8 g = pxl8_read_u8(stream);
        u8 b = pxl8_read_u8(stream);
        u8 a = pxl8_read_u8(stream);

        palette->colors[i] = (a << 24) | (b << 16) | (g << 8) | r;

        if (flags & 1) {
            u16 name_len = pxl8_read_u16(stream);
            pxl8_skip_bytes(stream, name_len);
        }
    }

    return PXL8_OK;
}

static pxl8_result parse_cel_chunk(pxl8_stream* stream, u32 chunk_size, pxl8_ase_cel* cel) {
    if (chunk_size < 9) {
        return PXL8_ERROR_ASE_MALFORMED_CHUNK;
    }

    cel->layer_index = pxl8_read_u16(stream);
    cel->x = pxl8_read_i16(stream);
    cel->y = pxl8_read_i16(stream);
    cel->opacity = pxl8_read_u8(stream);
    cel->cel_type = pxl8_read_u16(stream);
    pxl8_skip_bytes(stream, 7);

    if (cel->cel_type == 2) {
        if (chunk_size < 20) {
            return PXL8_ERROR_ASE_MALFORMED_CHUNK;
        }

        cel->width = pxl8_read_u16(stream);
        cel->height = pxl8_read_u16(stream);

        u32 pixel_data_size = cel->width * cel->height;
        u32 compressed_data_size = chunk_size - 20;

        cel->pixel_data = (u8*)malloc(pixel_data_size);
        if (!cel->pixel_data) {
            return PXL8_ERROR_OUT_OF_MEMORY;
        }

        const u8* compressed_data = pxl8_read_ptr(stream, compressed_data_size);
        mz_ulong dest_len = pixel_data_size;
        i32 result = mz_uncompress(cel->pixel_data, &dest_len, compressed_data, compressed_data_size);
        if (result != MZ_OK) {
            pxl8_error("Failed to decompress cel data: miniz error %d", result);
            free(cel->pixel_data);
            cel->pixel_data = NULL;
            return PXL8_ERROR_ASE_MALFORMED_CHUNK;
        }

        if (dest_len != pixel_data_size) {
            pxl8_warn("Decompressed size mismatch: expected %u, got %lu", pixel_data_size, dest_len);
        }
    }

    return PXL8_OK;
}

pxl8_result pxl8_ase_load(const char* filepath, pxl8_ase_file* ase_file) {
    if (!filepath || !ase_file) {
        return PXL8_ERROR_NULL_POINTER;
    }

    memset(ase_file, 0, sizeof(pxl8_ase_file));

    u8* file_data;
    size_t file_size;
    pxl8_result result = pxl8_io_read_binary_file(filepath, &file_data, &file_size);
    if (result != PXL8_OK) {
        return result;
    }

    if (file_size < 128) {
        pxl8_io_free_binary_data(file_data);
        return PXL8_ERROR_ASE_TRUNCATED_FILE;
    }

    pxl8_stream stream = pxl8_stream_create(file_data, (u32)file_size);

    result = parse_ase_header(&stream, &ase_file->header);
    if (result != PXL8_OK) {
        pxl8_io_free_binary_data(file_data);
        return result;
    }

    ase_file->frame_count = ase_file->header.frames;
    ase_file->frames = (pxl8_ase_frame*)calloc(ase_file->frame_count, sizeof(pxl8_ase_frame));
    if (!ase_file->frames) {
        pxl8_io_free_binary_data(file_data);
        return PXL8_ERROR_OUT_OF_MEMORY;
    }

    pxl8_stream_seek(&stream, 128);

    for (u16 frame_idx = 0; frame_idx < ase_file->header.frames; frame_idx++) {
        u32 frame_start = pxl8_stream_position(&stream);

        pxl8_ase_frame_header frame_header;
        frame_header.frame_bytes = pxl8_read_u32(&stream);
        frame_header.magic = pxl8_read_u16(&stream);
        u16 old_chunks = pxl8_read_u16(&stream);
        frame_header.duration = pxl8_read_u16(&stream);
        pxl8_skip_bytes(&stream, 2);

        if (frame_header.magic != PXL8_ASE_FRAME_MAGIC) {
            pxl8_error("Invalid frame magic: 0x%04X", frame_header.magic);
            result = PXL8_ERROR_ASE_INVALID_FRAME_MAGIC;
            break;
        }

        if (old_chunks == 0xFFFF || old_chunks == 0xFFFE) {
            frame_header.chunks = pxl8_read_u32(&stream);
        } else {
            frame_header.chunks = old_chunks;
            pxl8_skip_bytes(&stream, 4);
        }

        pxl8_ase_frame* frame = &ase_file->frames[frame_idx];
        frame->frame_id = frame_idx;
        frame->width = ase_file->header.width;
        frame->height = ase_file->header.height;
        frame->duration = frame_header.duration;

        u32 pixel_count = frame->width * frame->height;
        frame->pixels = (u8*)calloc(pixel_count, sizeof(u8));
        if (!frame->pixels) {
            result = PXL8_ERROR_OUT_OF_MEMORY;
            break;
        }

        for (u16 chunk_idx = 0; chunk_idx < frame_header.chunks; chunk_idx++) {
            u32 chunk_start = pxl8_stream_position(&stream);

            pxl8_ase_chunk_header chunk_header;
            chunk_header.chunk_size = pxl8_read_u32(&stream);
            chunk_header.chunk_type = pxl8_read_u16(&stream);

            if (chunk_header.chunk_size < 6 || chunk_header.chunk_size > frame_header.frame_bytes) {
                pxl8_error("Invalid chunk size %u in ASE file (frame_bytes=%u)",
                          chunk_header.chunk_size, frame_header.frame_bytes);
                result = PXL8_ERROR_ASE_MALFORMED_CHUNK;
                break;
            }

            switch (chunk_header.chunk_type) {
                case PXL8_ASE_CHUNK_OLD_PALETTE:
                    if (!ase_file->palette.colors) {
                        result = parse_old_palette_chunk(&stream, &ase_file->palette);
                    }
                    break;

                case PXL8_ASE_CHUNK_LAYER: {
                    ase_file->layers =
                        (pxl8_ase_layer*)realloc(ase_file->layers,
                            (ase_file->layer_count + 1) * sizeof(pxl8_ase_layer));
                    if (!ase_file->layers) {
                        result = PXL8_ERROR_OUT_OF_MEMORY;
                        break;
                    }

                    result = parse_layer_chunk(&stream, &ase_file->layers[ase_file->layer_count]);
                    if (result == PXL8_OK) {
                        ase_file->layer_count++;
                    }
                    break;
                }

                case PXL8_ASE_CHUNK_CEL: {
                    pxl8_ase_cel cel = {0};
                    result = parse_cel_chunk(&stream, chunk_header.chunk_size - 6, &cel);
                    if (result == PXL8_OK && cel.pixel_data) {
                        u32 copy_width = (cel.width < frame->width) ? cel.width : frame->width;
                        u32 copy_height = (cel.height < frame->height) ? cel.height : frame->height;

                        for (u32 y = 0; y < copy_height; y++) {
                            u32 src_offset = y * cel.width;
                            u32 dst_offset = (y + cel.y) * frame->width + cel.x;
                            if (dst_offset + copy_width <= pixel_count) {
                                for (u32 x = 0; x < copy_width; x++) {
                                    u8 src_pixel = cel.pixel_data[src_offset + x];
                                    bool is_transparent = false;

                                    if (src_pixel < ase_file->palette.entry_count && ase_file->palette.colors) {
                                        u32 color = ase_file->palette.colors[src_pixel];
                                        is_transparent = ((color >> 24) & 0xFF) == 0;
                                    }

                                    if (!is_transparent) {
                                        frame->pixels[dst_offset + x] = src_pixel;
                                    }
                                }
                            }
                        }
                        free(cel.pixel_data);
                    }
                    break;
                }

                case PXL8_ASE_CHUNK_PALETTE:
                    if (ase_file->palette.colors) {
                        free(ase_file->palette.colors);
                        ase_file->palette.colors = NULL;
                    }
                    result = parse_palette_chunk(&stream, &ase_file->palette);
                    break;

                default:
                    break;
            }

            if (result != PXL8_OK) break;

            pxl8_stream_seek(&stream, chunk_start + chunk_header.chunk_size);
        }

        if (result != PXL8_OK) break;

        pxl8_stream_seek(&stream, frame_start + frame_header.frame_bytes);
    }

    pxl8_io_free_binary_data(file_data);

    if (result != PXL8_OK) {
        pxl8_ase_destroy(ase_file);
    }

    return result;
}

void pxl8_ase_destroy(pxl8_ase_file* ase_file) {
    if (!ase_file) return;

    if (ase_file->frames) {
        for (u32 i = 0; i < ase_file->frame_count; i++) {
            if (ase_file->frames[i].pixels) {
                free(ase_file->frames[i].pixels);
            }
        }
        free(ase_file->frames);
    }

    if (ase_file->palette.colors) {
        free(ase_file->palette.colors);
    }

    if (ase_file->layers) {
        for (u32 i = 0; i < ase_file->layer_count; i++) {
            if (ase_file->layers[i].name) {
                free(ase_file->layers[i].name);
            }
        }
        free(ase_file->layers);
    }

    memset(ase_file, 0, sizeof(pxl8_ase_file));
}