diff --git a/Makefile.am b/Makefile.am index faf2252..18b12c6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -82,7 +82,10 @@ endif dist_man1_MANS = man/readstat.man man/extract_metadata.man -EXTRA_DIST = resources/datetime.sas7bdat +EXTRA_DIST = \ + resources/datetime.sas7bdat \ + resources/compression_type_2.sas7bdat \ + resources/compression_type_6.sas7bdat include_HEADERS = src/readstat.h diff --git a/resources/compression_type_2.sas7bdat b/resources/compression_type_2.sas7bdat new file mode 100644 index 0000000..fc1ab2d Binary files /dev/null and b/resources/compression_type_2.sas7bdat differ diff --git a/resources/compression_type_6.sas7bdat b/resources/compression_type_6.sas7bdat new file mode 100644 index 0000000..0f5f582 Binary files /dev/null and b/resources/compression_type_6.sas7bdat differ diff --git a/src/sas/readstat_sas.h b/src/sas/readstat_sas.h index 5d58910..6f8a6a8 100644 --- a/src/sas/readstat_sas.h +++ b/src/sas/readstat_sas.h @@ -123,8 +123,12 @@ typedef enum sas_subheader_type_e { #define SAS_COMPRESSION_NONE 0x00 #define SAS_COMPRESSION_TRUNC 0x01 +#define SAS_COMPRESSION_NONE_MOVED 0x02 +#define SAS_COMPRESSION_REFERENCE 0x03 #define SAS_COMPRESSION_ROW 0x04 #define SAS_COMPRESSION_DELETED_ROW 0x05 +#define SAS_COMPRESSION_ROW_MOVED 0x06 +#define SAS_COMPRESSION_MYSTERY 0x0d #define SAS_COMPRESSION_SIGNATURE_RLE "SASYZCRL" #define SAS_COMPRESSION_SIGNATURE_RDC "SASYZCR2" diff --git a/src/sas/readstat_sas7bdat_read.c b/src/sas/readstat_sas7bdat_read.c index cca57e8..6845654 100644 --- a/src/sas/readstat_sas7bdat_read.c +++ b/src/sas/readstat_sas7bdat_read.c @@ -43,6 +43,7 @@ typedef struct sas7bdat_ctx_s { readstat_io_t *io; int bswap; int did_submit_columns; + int requires_page_seek; uint32_t row_length; uint32_t page_row_count; @@ -965,7 +966,7 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { goto cleanup; } - if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { + if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC && shp_info.compression != SAS_COMPRESSION_REFERENCE) { if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { goto cleanup; } @@ -977,7 +978,9 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ goto cleanup; } } - } else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW) { + } else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW || + shp_info.compression == SAS_COMPRESSION_ROW_MOVED || shp_info.compression == SAS_COMPRESSION_MYSTERY || + shp_info.compression == SAS_COMPRESSION_NONE_MOVED) { /* void */ } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; @@ -1012,6 +1015,97 @@ static readstat_error_t sas7bdat_parse_deleted_row_bitmap(const char *page, cons return READSTAT_OK; } +static readstat_error_t sas7bdat_parse_moved_row(uint64_t page_index, uint64_t subheader_index, sas7bdat_ctx_t *ctx) { + readstat_error_t retval = READSTAT_OK; + readstat_io_t *io = ctx->io; + + const uint64_t page_size = ctx->page_size; + char *page = NULL; + + if (page_index >= ctx->page_count) { + retval = READSTAT_ERROR_PARSE; + goto cleanup; + } + + ctx->requires_page_seek = 1; + if (io->seek(ctx->header_size + page_index * page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) { + retval = READSTAT_ERROR_SEEK; + if (ctx->handle.error) { + snprintf(ctx->error_buf, sizeof(ctx->error_buf), "ReadStat: Failed to seek to position %" PRId64 + " (= %" PRId64 " + %" PRId64 "*%" PRId64 ")", + ctx->header_size + page_index * page_size, ctx->header_size, page_index, page_size); + ctx->handle.error(ctx->error_buf, ctx->user_ctx); + } + goto cleanup; + } + if ((page = readstat_malloc(page_size)) == NULL) { + retval = READSTAT_ERROR_MALLOC; + goto cleanup; + } + if (io->read(page, page_size, io->io_ctx) < page_size) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + + uint16_t page_type = sas_read2(&page[ctx->page_header_size - 8], ctx->bswap); + if ((page_type & SAS_PAGE_TYPE_MASK) == SAS_PAGE_TYPE_DATA || page_type & SAS_PAGE_TYPE_COMP) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + uint16_t subheader_count = sas_read2(&page[ctx->page_header_size - 4], ctx->bswap); + if (subheader_index >= subheader_count) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + uint64_t shp_offset = ctx->page_header_size + subheader_index * ctx->subheader_pointer_size; + if (shp_offset + ctx->subheader_pointer_size >= page_size) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + + const char *shp = &page[shp_offset]; + subheader_pointer_t shp_info = { 0 }; + if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { + goto cleanup; + } + if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { + goto cleanup; + } + if ((retval = sas7bdat_submit_columns_if_needed(ctx, 1)) != READSTAT_OK) { + goto cleanup; + } + + if (shp_info.compression == SAS_COMPRESSION_NONE_MOVED) { + sas_subheader_type_t subheader_type = sas7bdat_parse_subheader_type(page + shp_info.offset, ctx); + if (!shp_info.is_compressed_data || subheader_type != SAS_SUBHEADER_TYPE_DATA) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + if (shp_info.len != ctx->row_length) { + retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH; + goto cleanup; + } + if ((retval = sas7bdat_parse_single_row(page + shp_info.offset, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else if (shp_info.compression == SAS_COMPRESSION_ROW_MOVED) { + if ((retval = sas7bdat_parse_subheader_compressed(page + shp_info.offset, shp_info.len, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else { + retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; + goto cleanup; + } + +cleanup: + + if (page) { + free(page); + } + + return retval; +} + static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_size, sas7bdat_ctx_t *ctx) { uint16_t page_type; @@ -1041,7 +1135,13 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { goto cleanup; } - if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { + if (shp_info.len > 0 && shp_info.compression == SAS_COMPRESSION_REFERENCE) { + uint64_t page_index = shp_info.offset - 1; + uint64_t subheader_index = shp_info.len - 1; + if ((retval = sas7bdat_parse_moved_row(page_index, subheader_index, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { goto cleanup; } @@ -1076,6 +1176,9 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ if ((retval = sas7bdat_register_deleted_row(ctx)) != READSTAT_OK) { goto cleanup; } + } else if (shp_info.compression == SAS_COMPRESSION_ROW_MOVED || shp_info.compression == SAS_COMPRESSION_MYSTERY + || shp_info.compression == SAS_COMPRESSION_NONE_MOVED) { + /* void */ } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; goto cleanup; @@ -1256,6 +1359,19 @@ static readstat_error_t sas7bdat_parse_all_pages_pass2(sas7bdat_ctx_t *ctx) { if ((retval = sas7bdat_update_progress(ctx)) != READSTAT_OK) { goto cleanup; } + if (ctx->requires_page_seek) { + if (io->seek(ctx->header_size + i * ctx->page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) { + retval = READSTAT_ERROR_SEEK; + if (ctx->handle.error) { + snprintf(ctx->error_buf, sizeof(ctx->error_buf), "ReadStat: Failed to seek to position %" PRId64 + " (= %" PRId64 " + %" PRId64 "*%" PRId64 ")", + ctx->header_size + i * ctx->page_size, ctx->header_size, i, ctx->page_size); + ctx->handle.error(ctx->error_buf, ctx->user_ctx); + } + goto cleanup; + } + ctx->requires_page_seek = 0; + } if (io->read(ctx->page, ctx->page_size, io->io_ctx) < ctx->page_size) { retval = READSTAT_ERROR_READ; goto cleanup; diff --git a/src/test/test_list.h b/src/test/test_list.h index 9b78c98..0efe376 100644 --- a/src/test/test_list.h +++ b/src/test/test_list.h @@ -2383,6 +2383,80 @@ static rt_test_group_t _test_groups[] = { } } } + }, + + { + .resource_name = "compression_type_2.sas7bdat", + .label = "SAS file with moved rows", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE, + .rows = 5, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .skip_value_comparison = 1 + } + } + }, + + { + .resource_name = "compression_type_6.sas7bdat", + .label = "SAS file with moved rows", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS, + .rows = 5, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .skip_value_comparison = 1 + } + } } } } diff --git a/src/test/test_read.c b/src/test/test_read.c index 1bd9389..f46da39 100644 --- a/src/test/test_read.c +++ b/src/test/test_read.c @@ -265,14 +265,16 @@ static int handle_value(int obs_index, readstat_variable_t *variable, readstat_v rt_column_t *column = &rt_ctx->file->columns[rt_ctx->var_index]; - if (column->type == READSTAT_TYPE_STRING_REF) { - push_error_if_strings_differ(rt_ctx, - rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])], - readstat_string_value(value), "String ref values"); - } else { - push_error_if_values_differ(rt_ctx, - column->values[file_obs_index], - value, "Data values"); + if (!column->skip_value_comparison) { + if (column->type == READSTAT_TYPE_STRING_REF) { + push_error_if_strings_differ(rt_ctx, + rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])], + readstat_string_value(value), "String ref values"); + } else { + push_error_if_values_differ(rt_ctx, + column->values[file_obs_index], + value, "Data values"); + } } return READSTAT_HANDLER_OK; diff --git a/src/test/test_types.h b/src/test/test_types.h index 5783933..d086284 100644 --- a/src/test/test_types.h +++ b/src/test/test_types.h @@ -42,6 +42,8 @@ typedef struct rt_column_s { long missing_ranges_count; char label_set[RT_MAX_STRING]; + + int skip_value_comparison; } rt_column_t; typedef struct rt_test_file_s {