Skip to content

Support moved rows in SAS data files #365

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hpoettker wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Support moved rows in SAS data files #365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,10 @@ endif

dist_man1_MANS = man/readstat.man man/extract_metadata.man

EXTRA_DIST = resources/datetime.sas7bdat
EXTRA_DIST = \
resources/datetime.sas7bdat \
resources/compression_type_2.sas7bdat \
resources/compression_type_6.sas7bdat

include_HEADERS = src/readstat.h

Expand Down
Binary file added resources/compression_type_2.sas7bdat
View file
Open in desktop
Binary file not shown.
Binary file added resources/compression_type_6.sas7bdat
View file
Open in desktop
Binary file not shown.
4 changes: 4 additions & 0 deletions src/sas/readstat_sas.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,12 @@ typedef enum sas_subheader_type_e {

#define SAS_COMPRESSION_NONE 0x00
#define SAS_COMPRESSION_TRUNC 0x01
#define SAS_COMPRESSION_NONE_MOVED 0x02
#define SAS_COMPRESSION_REFERENCE 0x03
#define SAS_COMPRESSION_ROW 0x04
#define SAS_COMPRESSION_DELETED_ROW 0x05
#define SAS_COMPRESSION_ROW_MOVED 0x06
#define SAS_COMPRESSION_MYSTERY 0x0d

#define SAS_COMPRESSION_SIGNATURE_RLE "SASYZCRL"
#define SAS_COMPRESSION_SIGNATURE_RDC "SASYZCR2"
Expand Down
122 changes: 119 additions & 3 deletions src/sas/readstat_sas7bdat_read.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ typedef struct sas7bdat_ctx_s {
readstat_io_t *io;
int bswap;
int did_submit_columns;
int requires_page_seek;

uint32_t row_length;
uint32_t page_row_count;
Expand Down Expand Up @@ -965,7 +966,7 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_
if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) {
goto cleanup;
}
if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) {
if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC && shp_info.compression != SAS_COMPRESSION_REFERENCE) {
if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) {
goto cleanup;
}
Expand All @@ -977,7 +978,9 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_
goto cleanup;
}
}
} else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW) {
} else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW ||
shp_info.compression == SAS_COMPRESSION_ROW_MOVED || shp_info.compression == SAS_COMPRESSION_MYSTERY ||
shp_info.compression == SAS_COMPRESSION_NONE_MOVED) {
/* void */
} else {
retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION;
Expand Down Expand Up @@ -1012,6 +1015,97 @@ static readstat_error_t sas7bdat_parse_deleted_row_bitmap(const char *page, cons
return READSTAT_OK;
}

static readstat_error_t sas7bdat_parse_moved_row(uint64_t page_index, uint64_t subheader_index, sas7bdat_ctx_t *ctx) {
readstat_error_t retval = READSTAT_OK;
readstat_io_t *io = ctx->io;

const uint64_t page_size = ctx->page_size;
char *page = NULL;

if (page_index >= ctx->page_count) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}

ctx->requires_page_seek = 1;
if (io->seek(ctx->header_size + page_index * page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) {
retval = READSTAT_ERROR_SEEK;
if (ctx->handle.error) {
snprintf(ctx->error_buf, sizeof(ctx->error_buf), "ReadStat: Failed to seek to position %" PRId64
" (= %" PRId64 " + %" PRId64 "*%" PRId64 ")",
ctx->header_size + page_index * page_size, ctx->header_size, page_index, page_size);
ctx->handle.error(ctx->error_buf, ctx->user_ctx);
}
goto cleanup;
}
if ((page = readstat_malloc(page_size)) == NULL) {
retval = READSTAT_ERROR_MALLOC;
goto cleanup;
}
if (io->read(page, page_size, io->io_ctx) < page_size) {
retval = READSTAT_ERROR_READ;
goto cleanup;
}

uint16_t page_type = sas_read2(&page[ctx->page_header_size - 8], ctx->bswap);
if ((page_type & SAS_PAGE_TYPE_MASK) == SAS_PAGE_TYPE_DATA || page_type & SAS_PAGE_TYPE_COMP) {
retval = READSTAT_ERROR_READ;
goto cleanup;
}
uint16_t subheader_count = sas_read2(&page[ctx->page_header_size - 4], ctx->bswap);
if (subheader_index >= subheader_count) {
retval = READSTAT_ERROR_READ;
goto cleanup;
}
uint64_t shp_offset = ctx->page_header_size + subheader_index * ctx->subheader_pointer_size;
if (shp_offset + ctx->subheader_pointer_size >= page_size) {
retval = READSTAT_ERROR_READ;
goto cleanup;
}

const char *shp = &page[shp_offset];
subheader_pointer_t shp_info = { 0 };
if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) {
goto cleanup;
}
if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) {
goto cleanup;
}
if ((retval = sas7bdat_submit_columns_if_needed(ctx, 1)) != READSTAT_OK) {
goto cleanup;
}

if (shp_info.compression == SAS_COMPRESSION_NONE_MOVED) {
sas_subheader_type_t subheader_type = sas7bdat_parse_subheader_type(page + shp_info.offset, ctx);
if (!shp_info.is_compressed_data || subheader_type != SAS_SUBHEADER_TYPE_DATA) {
retval = READSTAT_ERROR_READ;
goto cleanup;
}
if (shp_info.len != ctx->row_length) {
retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
goto cleanup;
}
if ((retval = sas7bdat_parse_single_row(page + shp_info.offset, ctx)) != READSTAT_OK) {
goto cleanup;
}
} else if (shp_info.compression == SAS_COMPRESSION_ROW_MOVED) {
if ((retval = sas7bdat_parse_subheader_compressed(page + shp_info.offset, shp_info.len, ctx)) != READSTAT_OK) {
goto cleanup;
}
} else {
retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION;
goto cleanup;
}

cleanup:

if (page) {
free(page);
}

return retval;
}

static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_size, sas7bdat_ctx_t *ctx) {
uint16_t page_type;

Expand Down Expand Up @@ -1041,7 +1135,13 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_
if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) {
goto cleanup;
}
if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) {
if (shp_info.len > 0 && shp_info.compression == SAS_COMPRESSION_REFERENCE) {
uint64_t page_index = shp_info.offset - 1;
uint64_t subheader_index = shp_info.len - 1;
if ((retval = sas7bdat_parse_moved_row(page_index, subheader_index, ctx)) != READSTAT_OK) {
goto cleanup;
}
} else if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) {
if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) {
goto cleanup;
}
Expand Down Expand Up @@ -1076,6 +1176,9 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_
if ((retval = sas7bdat_register_deleted_row(ctx)) != READSTAT_OK) {
goto cleanup;
}
} else if (shp_info.compression == SAS_COMPRESSION_ROW_MOVED || shp_info.compression == SAS_COMPRESSION_MYSTERY
|| shp_info.compression == SAS_COMPRESSION_NONE_MOVED) {
/* void */
} else {
retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION;
goto cleanup;
Expand Down Expand Up @@ -1256,6 +1359,19 @@ static readstat_error_t sas7bdat_parse_all_pages_pass2(sas7bdat_ctx_t *ctx) {
if ((retval = sas7bdat_update_progress(ctx)) != READSTAT_OK) {
goto cleanup;
}
if (ctx->requires_page_seek) {
if (io->seek(ctx->header_size + i * ctx->page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) {
retval = READSTAT_ERROR_SEEK;
if (ctx->handle.error) {
snprintf(ctx->error_buf, sizeof(ctx->error_buf), "ReadStat: Failed to seek to position %" PRId64
" (= %" PRId64 " + %" PRId64 "*%" PRId64 ")",
ctx->header_size + i * ctx->page_size, ctx->header_size, i, ctx->page_size);
ctx->handle.error(ctx->error_buf, ctx->user_ctx);
}
goto cleanup;
}
ctx->requires_page_seek = 0;
}
if (io->read(ctx->page, ctx->page_size, io->io_ctx) < ctx->page_size) {
retval = READSTAT_ERROR_READ;
goto cleanup;
Expand Down
74 changes: 74 additions & 0 deletions src/test/test_list.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2383,6 +2383,80 @@ static rt_test_group_t _test_groups[] = {
}
}
}
},

{
.resource_name = "compression_type_2.sas7bdat",
.label = "SAS file with moved rows",
.test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE,
.rows = 5,
.columns_count = 3,
.columns = {
{
.name = "id",
.type = READSTAT_TYPE_DOUBLE,
.values = {
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } }
}
},
{
.name = "category",
.type = READSTAT_TYPE_STRING,
.values = {
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } },
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } },
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } },
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } },
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } }
}
},
{
.name = "payload_to_update",
.type = READSTAT_TYPE_STRING,
.skip_value_comparison = 1
}
}
},

{
.resource_name = "compression_type_6.sas7bdat",
.label = "SAS file with moved rows",
.test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS,
.rows = 5,
.columns_count = 3,
.columns = {
{
.name = "id",
.type = READSTAT_TYPE_DOUBLE,
.values = {
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } }
}
},
{
.name = "category",
.type = READSTAT_TYPE_STRING,
.values = {
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } },
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } },
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } },
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } },
{ .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } }
}
},
{
.name = "payload_to_update",
.type = READSTAT_TYPE_STRING,
.skip_value_comparison = 1
}
}
}
}
}
Expand Down
18 changes: 10 additions & 8 deletions src/test/test_read.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,14 +265,16 @@ static int handle_value(int obs_index, readstat_variable_t *variable, readstat_v

rt_column_t *column = &rt_ctx->file->columns[rt_ctx->var_index];

if (column->type == READSTAT_TYPE_STRING_REF) {
push_error_if_strings_differ(rt_ctx,
rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])],
readstat_string_value(value), "String ref values");
} else {
push_error_if_values_differ(rt_ctx,
column->values[file_obs_index],
value, "Data values");
if (!column->skip_value_comparison) {
if (column->type == READSTAT_TYPE_STRING_REF) {
push_error_if_strings_differ(rt_ctx,
rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])],
readstat_string_value(value), "String ref values");
} else {
push_error_if_values_differ(rt_ctx,
column->values[file_obs_index],
value, "Data values");
}
}

return READSTAT_HANDLER_OK;
Expand Down
2 changes: 2 additions & 0 deletions src/test/test_types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ typedef struct rt_column_s {
long missing_ranges_count;

char label_set[RT_MAX_STRING];

int skip_value_comparison;
} rt_column_t;

typedef struct rt_test_file_s {
Expand Down
Loading