add system dependent strings support.

this hopefully allows to use .mo files generated by gettext-0.12 or later.
This commit is contained in:
tshiozak 2004-09-23 21:35:27 +00:00
parent ce2c46e673
commit 922c0d0065
4 changed files with 531 additions and 35 deletions

View File

@ -1,9 +1,10 @@
# $NetBSD: Makefile,v 1.4 2004/09/23 16:44:26 tshiozak Exp $
# $NetBSD: Makefile,v 1.5 2004/09/23 21:35:27 tshiozak Exp $
.include <bsd.own.mk>
LIB= intl
SRCS= gettext.c textdomain.c gettext_iconv.c gettext_dummy.c strhash.c
SRCS= gettext.c textdomain.c gettext_iconv.c gettext_dummy.c strhash.c \
sysdep.c
INCS= libintl.h
INCSDIR=/usr/include

View File

@ -1,4 +1,4 @@
/* $NetBSD: gettext.c,v 1.19 2004/09/23 16:44:26 tshiozak Exp $ */
/* $NetBSD: gettext.c,v 1.20 2004/09/23 21:35:27 tshiozak Exp $ */
/*-
* Copyright (c) 2000, 2001 Citrus Project,
@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
__RCSID("$NetBSD: gettext.c,v 1.19 2004/09/23 16:44:26 tshiozak Exp $");
__RCSID("$NetBSD: gettext.c,v 1.20 2004/09/23 21:35:27 tshiozak Exp $");
#include <sys/param.h>
#include <sys/stat.h>
@ -300,6 +300,196 @@ validate(arg, mohandle)
return 1;
}
/*
* calculate the step value if the hash value is conflicted.
*/
static __inline u_int32_t
calc_collision_step(u_int32_t hashval, u_int32_t hashsize)
{
_DIAGASSERT(hashsize>2);
return (hashval % (hashsize - 2)) + 1;
}
/*
* calculate the next index while conflicting.
*/
static __inline u_int32_t
calc_next_index(u_int32_t curidx, u_int32_t hashsize, u_int32_t step)
{
return curidx+step - (curidx >= hashsize-step ? hashsize : 0);
}
static int
get_sysdep_string_table(struct mosysdepstr_h **table_h,
u_int32_t *ofstable, uint32_t nstrings,
u_int32_t magic, char *base)
{
int i, j;
int count;
size_t l;
struct mosysdepstr *table;
for (i=0; i<nstrings; i++) {
/* get mosysdepstr record */
/* LINTED: ignore the alignment problem. */
table = (struct mosysdepstr *)(base + flip(ofstable[i], magic));
/* count number of segments */
count = 0;
while (flip(table->segs[count++].ref, magic) != MO_LASTSEG)
;
/* get table */
l = sizeof(struct mosysdepstr_h) +
sizeof(struct mosysdepsegentry_h) * (count-1);
table_h[i] = (struct mosysdepstr_h *)malloc(l);
if (!table_h[i])
return -1;
memset(table_h[i], 0, l);
table_h[i]->off = (const char *)(base + flip(table->off, magic));
for (j=0; j<count; j++) {
table_h[i]->segs[j].len =
flip(table->segs[j].len, magic);
table_h[i]->segs[j].ref =
flip(table->segs[j].ref, magic);
}
/* LINTED: ignore the alignment problem. */
table = (struct mosysdepstr *)&table->segs[count];
}
return 0;
}
static int
expand_sysdep(struct mohandle *mohandle, struct mosysdepstr_h *str)
{
int i;
const char *src;
char *dst;
/* check whether already expanded */
if (str->expanded)
return 0;
/* calc total length */
str->expanded_len = 1;
for (i=0; /*CONSTCOND*/1; i++) {
str->expanded_len += str->segs[i].len;
if (str->segs[i].ref == MO_LASTSEG)
break;
str->expanded_len +=
mohandle->mo.mo_sysdep_segs[str->segs[i].ref].len;
}
/* expand */
str->expanded = malloc(str->expanded_len);
if (!str->expanded)
return -1;
src = str->off;
dst = str->expanded;
for (i=0; /*CONSTCOND*/1; i++) {
memcpy(dst, src, str->segs[i].len);
src += str->segs[i].len;
dst += str->segs[i].len;
if (str->segs[i].ref == MO_LASTSEG)
break;
memcpy(dst, mohandle->mo.mo_sysdep_segs[str->segs[i].ref].str,
mohandle->mo.mo_sysdep_segs[str->segs[i].ref].len);
dst += mohandle->mo.mo_sysdep_segs[str->segs[i].ref].len;
}
*dst = '\0';
return 0;
}
static void
insert_to_hash(u_int32_t *htable, u_int32_t hsize, const char *str,
u_int32_t ref)
{
u_int32_t hashval, idx, step;
hashval = __intl_string_hash(str);
step = calc_collision_step(hashval, hsize);
idx = hashval % hsize;
while (htable[idx])
idx = calc_next_index(idx, hsize, step);
htable[idx] = ref;
}
static int
setup_sysdep_stuffs(struct mo *mo, struct mohandle *mohandle, char *base)
{
u_int32_t magic;
struct moentry *stable;
size_t l;
int i;
char *v;
u_int32_t *ofstable;
magic = mo->mo_magic;
mohandle->mo.mo_sysdep_nsegs = flip(mo->mo_sysdep_nsegs, magic);
mohandle->mo.mo_sysdep_nstring = flip(mo->mo_sysdep_nstring, magic);
if (mohandle->mo.mo_sysdep_nstring == 0)
return 0;
/* check hash size */
if (mohandle->mo.mo_hsize <= 2 ||
mohandle->mo.mo_hsize <
(mohandle->mo.mo_nstring + mohandle->mo.mo_sysdep_nstring))
return -1;
/* get sysdep segments */
l = sizeof(struct mosysdepsegs_h *) * mohandle->mo.mo_sysdep_nsegs;
mohandle->mo.mo_sysdep_segs = (struct mosysdepsegs_h *)malloc(l);
if (!mohandle->mo.mo_sysdep_segs)
return -1;
/* LINTED: ignore the alignment problem. */
stable = (struct moentry *)(base + flip(mo->mo_sysdep_segoff, magic));
for (i=0; i<mohandle->mo.mo_sysdep_nsegs; i++) {
v = base + flip(stable[i].off, magic);
mohandle->mo.mo_sysdep_segs[i].str =
__intl_sysdep_get_string_by_tag(
v,
&mohandle->mo.mo_sysdep_segs[i].len);
}
/* get sysdep string table */
mohandle->mo.mo_sysdep_otable =
(struct mosysdepstr_h **)calloc(mohandle->mo.mo_sysdep_nstring,
sizeof(struct mosysdepstr_h *));
if (!mohandle->mo.mo_sysdep_otable)
return -1;
/* LINTED: ignore the alignment problem. */
ofstable = (u_int32_t *)(base + flip(mo->mo_sysdep_otable, magic));
if (get_sysdep_string_table(mohandle->mo.mo_sysdep_otable, ofstable,
mohandle->mo.mo_sysdep_nstring, magic,
base))
return -1;
mohandle->mo.mo_sysdep_ttable =
(struct mosysdepstr_h **)calloc(mohandle->mo.mo_sysdep_nstring,
sizeof(struct mosysdepstr_h *));
if (!mohandle->mo.mo_sysdep_ttable)
return -1;
/* LINTED: ignore the alignment problem. */
ofstable = (u_int32_t *)(base + flip(mo->mo_sysdep_ttable, magic));
if (get_sysdep_string_table(mohandle->mo.mo_sysdep_ttable, ofstable,
mohandle->mo.mo_sysdep_nstring, magic,
base))
return -1;
/* update hash */
for (i=0; i<mohandle->mo.mo_sysdep_nstring; i++) {
if (expand_sysdep(mohandle, mohandle->mo.mo_sysdep_otable[i]))
return -1;
insert_to_hash(mohandle->mo.mo_htable,
mohandle->mo.mo_hsize,
mohandle->mo.mo_sysdep_otable[i]->expanded,
(i+1) | MO_HASH_SYSDEP_MASK);
}
return 0;
}
int
mapit(path, db)
const char *path;
@ -308,7 +498,7 @@ mapit(path, db)
int fd;
struct stat st;
char *base;
u_int32_t magic, revision;
u_int32_t magic, revision, flags = 0;
struct moentry *otable, *ttable;
const u_int32_t *htable;
struct moentry_h *p;
@ -346,10 +536,10 @@ mapit(path, db)
}
switch (flip(revision, magic)) {
case MO_MAKE_REV(0, 0):
#if 0
break;
case MO_MAKE_REV(0, 1):
case MO_MAKE_REV(1, 1):
#endif
flags |= MO_F_SYSDEP;
break;
default:
close(fd);
@ -372,6 +562,7 @@ mapit(path, db)
mohandle->mo.mo_revision = flip(mo->mo_revision, magic);
mohandle->mo.mo_nstring = flip(mo->mo_nstring, magic);
mohandle->mo.mo_hsize = flip(mo->mo_hsize, magic);
mohandle->mo.mo_flags = flags;
/* validate otable/ttable */
/* LINTED: ignore the alignment problem. */
@ -466,12 +657,35 @@ mapit(path, db)
* the *.mo file as we cannot support it.
*/
/* system dependent string support */
if ((mohandle->mo.mo_flags & MO_F_SYSDEP) != 0) {
if (setup_sysdep_stuffs(mo, mohandle, base)) {
unmapit(db);
goto fail;
}
}
return 0;
fail:
return -1;
}
static void
free_sysdep_table(struct mosysdepstr_h **table, u_int32_t nstring)
{
u_int32_t i;
for (i=0; i<nstring; i++) {
if (table[i]) {
if (table[i]->expanded)
free(table[i]->expanded);
free(table[i]);
}
}
free(table);
}
static int
unmapit(db)
struct domainbinding *db;
@ -490,29 +704,20 @@ unmapit(db)
free(mohandle->mo.mo_charset);
if (mohandle->mo.mo_htable)
free(mohandle->mo.mo_htable);
if (mohandle->mo.mo_sysdep_segs)
free(mohandle->mo.mo_sysdep_segs);
if (mohandle->mo.mo_sysdep_otable) {
free_sysdep_table(mohandle->mo.mo_sysdep_otable,
mohandle->mo.mo_sysdep_nstring);
}
if (mohandle->mo.mo_sysdep_ttable) {
free_sysdep_table(mohandle->mo.mo_sysdep_ttable,
mohandle->mo.mo_sysdep_nstring);
}
memset(&mohandle->mo, 0, sizeof(mohandle->mo));
return 0;
}
/*
* calculate the step value if the hash value is conflicted.
*/
static __inline u_int32_t
calc_collision_step(u_int32_t hashval, u_int32_t hashsize)
{
_DIAGASSERT(hashsize>2);
return (hashval % (hashsize - 2)) + 1;
}
/*
* calculate the next index while conflicting.
*/
static __inline u_int32_t
calc_next_index(u_int32_t curidx, u_int32_t hashsize, u_int32_t step)
{
return curidx+step - (curidx >= hashsize-step ? hashsize : 0);
}
/* ARGSUSED */
static const char *
lookup_hash(msgid, db)
@ -522,6 +727,7 @@ lookup_hash(msgid, db)
struct mohandle *mohandle = &db->mohandle;
u_int32_t idx, hashval, step, strno;
size_t len;
struct mosysdepstr_h *sysdep_otable, *sysdep_ttable;
if (mohandle->mo.mo_hsize <= 2 || mohandle->mo.mo_htable == NULL)
return NULL;
@ -537,10 +743,26 @@ lookup_hash(msgid, db)
return NULL;
}
strno--;
if (len <= mohandle->mo.mo_otable[strno].len &&
!strcmp(msgid, mohandle->mo.mo_otable[strno].off)) {
/* hit */
return mohandle->mo.mo_ttable[strno].off;
if ((strno & MO_HASH_SYSDEP_MASK) == 0) {
/* system independent strings */
if (len <= mohandle->mo.mo_otable[strno].len &&
!strcmp(msgid, mohandle->mo.mo_otable[strno].off)) {
/* hit */
return mohandle->mo.mo_ttable[strno].off;
}
} else {
/* system dependent strings */
strno &= ~MO_HASH_SYSDEP_MASK;
sysdep_otable = mohandle->mo.mo_sysdep_otable[strno];
sysdep_ttable = mohandle->mo.mo_sysdep_ttable[strno];
if (len <= sysdep_otable->expanded_len &&
!strcmp(msgid, sysdep_otable->expanded)) {
/* hit */
if (expand_sysdep(mohandle, sysdep_ttable))
/* memory exhausted */
return NULL;
return sysdep_ttable->expanded;
}
}
idx = calc_next_index(idx, mohandle->mo.mo_hsize, step);
}

View File

@ -1,4 +1,4 @@
/* $NetBSD: libintl_local.h,v 1.8 2004/09/23 16:44:26 tshiozak Exp $ */
/* $NetBSD: libintl_local.h,v 1.9 2004/09/23 21:35:27 tshiozak Exp $ */
/*-
* Copyright (c) 2000, 2001 Citrus Project,
@ -61,10 +61,17 @@ struct moentry {
u_int32_t off; /* offset of \0-terminated string */
} __attribute__((__packed__));
struct mosysdepstr
{
u_int32_t off; /* offset of seed text */
struct moentry segs[1]; /* text segments */
struct mosysdepsegentry {
u_int32_t len; /* length of this part */
u_int32_t ref; /* reference number of the sysdep string,
* concatenated just after this segment.
*/
} __attribute__((__packed__));
#define MO_LASTSEG (0xFFFFFFFF)
struct mosysdepstr {
u_int32_t off; /* offset of seed text */
struct mosysdepsegentry segs[1]; /* text segments */
} __attribute__((__packed__));
/* libintl internal data format */
@ -73,6 +80,23 @@ struct moentry_h {
char *off; /* offset of \0-terminated string */
};
struct mosysdepsegs_h {
const char *str;
size_t len;
};
struct mosysdepsegentry_h {
u_int32_t len;
u_int32_t ref;
};
struct mosysdepstr_h {
const char *off; /* offset of the base string */
char *expanded; /* expanded string */
size_t expanded_len; /* length of expanded string */
struct mosysdepsegentry_h segs[1]; /* text segments */
};
struct mo_h {
u_int32_t mo_magic; /* determines endian */
u_int32_t mo_revision; /* file format revision: 0 */
@ -83,8 +107,20 @@ struct mo_h {
char *mo_charset;
u_int32_t mo_hsize; /* S: size of hashing table */
u_int32_t *mo_htable; /* H: hashing table */
#define MO_HASH_SYSDEP_MASK 0x80000000 /* means sysdep entry */
u_int32_t mo_flags;
#define MO_F_SYSDEP 0x00000001 /* enable sysdep string support */
/* system dependent string support */
u_int32_t mo_sysdep_nsegs; /* number of sysdep segments */
u_int32_t mo_sysdep_nstring; /* number of sysdep strings */
struct mosysdepsegs_h *mo_sysdep_segs; /* sysdep segment table */
struct mosysdepstr_h **mo_sysdep_otable; /* original text */
struct mosysdepstr_h **mo_sysdep_ttable; /* translated text */
};
struct mohandle {
void *addr; /* mmap'ed region */
size_t len;
@ -105,4 +141,5 @@ extern char __current_domainname[PATH_MAX];
__BEGIN_DECLS
const char *__gettext_iconv __P((const char *, struct domainbinding *));
u_int32_t __intl_string_hash __P((const char *));
const char *__intl_sysdep_get_string_by_tag __P((const char *, size_t *));
__END_DECLS

236
lib/libintl/sysdep.c Normal file
View File

@ -0,0 +1,236 @@
/* $NetBSD: sysdep.c,v 1.1 2004/09/23 21:35:27 tshiozak Exp $ */
/*-
* Copyright (c)2004 Citrus Project,
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#if defined(LIBC_SCCS) && !defined(lint)
__RCSID("$NetBSD: sysdep.c,v 1.1 2004/09/23 21:35:27 tshiozak Exp $");
#endif /* LIBC_SCCS and not lint */
#include <sys/param.h>
#include <sys/types.h>
#include <assert.h>
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
#include "libintl_local.h"
struct sysdep_pair
{
const char *tag;
const char *string;
size_t len;
};
#define T_(tag) { #tag, tag, sizeof (tag)-1 }
#define numof(a) (sizeof (a) / sizeof ((a)[0]))
const struct sysdep_pair sysdep_pair_table[] = {
/* this table must be sorted in the dictionary order. */
T_(PRIX16),
T_(PRIX32),
T_(PRIX64),
T_(PRIX8),
T_(PRIXFAST16),
T_(PRIXFAST32),
T_(PRIXFAST64),
T_(PRIXFAST8),
T_(PRIXLEAST16),
T_(PRIXLEAST32),
T_(PRIXLEAST64),
T_(PRIXLEAST8),
T_(PRIXMAX),
T_(PRIXPTR),
T_(PRId16),
T_(PRId32),
T_(PRId64),
T_(PRId8),
T_(PRIdFAST16),
T_(PRIdFAST32),
T_(PRIdFAST64),
T_(PRIdFAST8),
T_(PRIdLEAST16),
T_(PRIdLEAST32),
T_(PRIdLEAST64),
T_(PRIdLEAST8),
T_(PRIdMAX),
T_(PRIdPTR),
T_(PRIi16),
T_(PRIi32),
T_(PRIi64),
T_(PRIi8),
T_(PRIiFAST16),
T_(PRIiFAST32),
T_(PRIiFAST64),
T_(PRIiFAST8),
T_(PRIiLEAST16),
T_(PRIiLEAST32),
T_(PRIiLEAST64),
T_(PRIiLEAST8),
T_(PRIiMAX),
T_(PRIiPTR),
T_(PRIo16),
T_(PRIo32),
T_(PRIo64),
T_(PRIo8),
T_(PRIoFAST16),
T_(PRIoFAST32),
T_(PRIoFAST64),
T_(PRIoFAST8),
T_(PRIoLEAST16),
T_(PRIoLEAST32),
T_(PRIoLEAST64),
T_(PRIoLEAST8),
T_(PRIoMAX),
T_(PRIoPTR),
T_(PRIu16),
T_(PRIu32),
T_(PRIu64),
T_(PRIu8),
T_(PRIuFAST16),
T_(PRIuFAST32),
T_(PRIuFAST64),
T_(PRIuFAST8),
T_(PRIuLEAST16),
T_(PRIuLEAST32),
T_(PRIuLEAST64),
T_(PRIuLEAST8),
T_(PRIuMAX),
T_(PRIuPTR),
T_(PRIx16),
T_(PRIx32),
T_(PRIx64),
T_(PRIx8),
T_(PRIxFAST16),
T_(PRIxFAST32),
T_(PRIxFAST64),
T_(PRIxFAST8),
T_(PRIxLEAST16),
T_(PRIxLEAST32),
T_(PRIxLEAST64),
T_(PRIxLEAST8),
T_(PRIxMAX),
T_(PRIxPTR),
T_(SCNd16),
T_(SCNd32),
T_(SCNd64),
T_(SCNd8),
T_(SCNdFAST16),
T_(SCNdFAST32),
T_(SCNdFAST64),
T_(SCNdFAST8),
T_(SCNdLEAST16),
T_(SCNdLEAST32),
T_(SCNdLEAST64),
T_(SCNdLEAST8),
T_(SCNdMAX),
T_(SCNdPTR),
T_(SCNi16),
T_(SCNi32),
T_(SCNi64),
T_(SCNi8),
T_(SCNiFAST16),
T_(SCNiFAST32),
T_(SCNiFAST64),
T_(SCNiFAST8),
T_(SCNiLEAST16),
T_(SCNiLEAST32),
T_(SCNiLEAST64),
T_(SCNiLEAST8),
T_(SCNiMAX),
T_(SCNiPTR),
T_(SCNo16),
T_(SCNo32),
T_(SCNo64),
T_(SCNo8),
T_(SCNoFAST16),
T_(SCNoFAST32),
T_(SCNoFAST64),
T_(SCNoFAST8),
T_(SCNoLEAST16),
T_(SCNoLEAST32),
T_(SCNoLEAST64),
T_(SCNoLEAST8),
T_(SCNoMAX),
T_(SCNoPTR),
T_(SCNu16),
T_(SCNu32),
T_(SCNu64),
T_(SCNu8),
T_(SCNuFAST16),
T_(SCNuFAST32),
T_(SCNuFAST64),
T_(SCNuFAST8),
T_(SCNuLEAST16),
T_(SCNuLEAST32),
T_(SCNuLEAST64),
T_(SCNuLEAST8),
T_(SCNuMAX),
T_(SCNuPTR),
T_(SCNx16),
T_(SCNx32),
T_(SCNx64),
T_(SCNx8),
T_(SCNxFAST16),
T_(SCNxFAST32),
T_(SCNxFAST64),
T_(SCNxFAST8),
T_(SCNxLEAST16),
T_(SCNxLEAST32),
T_(SCNxLEAST64),
T_(SCNxLEAST8),
T_(SCNxMAX),
T_(SCNxPTR)
};
static int
cmp_sysdep_tag(const void *tag, const void *elem)
{
const struct sysdep_pair *pair = elem;
return strcmp(tag, pair->tag);
}
const char *
__intl_sysdep_get_string_by_tag(const char *tag, size_t *rlen)
{
const struct sysdep_pair *found;
found = bsearch(tag, sysdep_pair_table, numof(sysdep_pair_table),
sizeof(sysdep_pair_table[0]), &cmp_sysdep_tag);
if (found) {
if (rlen)
*rlen = found->len;
return found->string;
}
if (rlen)
*rlen = 0;
return "";
}