Manpage of CHARSETS
.\" Automatically generated by Pod::Man v1.37, Pod::Parser v1.14
.\"
.\" Standard preamble:
.\" ========================================================================
.de Sh \" Subsection heading
.br
.if t .Sp
.ne 5
.PP
\fB\\$1\fR
.PP
..
.de Sp \" Vertical space (when we can't use .PP)
.if t .sp .5v
.if n .sp
..
.de Vb \" Begin verbatim text
.ft CW
.nf
.ne \\$1
..
.de Ve \" End verbatim text
.ft R
.fi
..
.\" Set up some character translations and predefined strings. \*(-- will
.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
.\" double quote, and \*(R" will give a right double quote. | will give a
.\" real vertical bar. \*(C+ will give a nicer C++. Capital omega is used to
.\" do unbreakable dashes and therefore won't be available. \*(C` and \*(C'
.\" expand to `' in nroff, nothing in troff, for use with C<>.
.tr \(*W-|\(bv\*(Tr
.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
.ie n \{\
. ds -- \(*W-
. ds PI pi
. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch
. ds L" ""
. ds R" ""
. ds C` ""
. ds C' ""
'br\}
.el\{\
. ds -- \|\(em\|
. ds PI \(*p
. ds L" ``
. ds R" ''
'br\}
.\"
.\" If the F register is turned on, we'll generate index entries on stderr for
.\" titles (.TH), headers (.SH), subsections (.Sh), items (.Ip), and index
.\" entries marked with X<> in POD. Of course, you'll have to process the
.\" output yourself in some meaningful fashion.
.if \nF \{\
. de IX
. tm Index:\\$1\t\\n%\t"\\$2"
..
. nr % 0
. rr F
.\}
.\"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.hy 0
.if n .na
.\"
.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
.\" Fear. Run. Save yourself. No user-serviceable parts.
. \" fudge factors for nroff and troff
.if n \{\
. ds #H 0
. ds #V .8m
. ds #F .3m
. ds #[ \f1
. ds #] \fP
.\}
.if t \{\
. ds #H ((1u-(\\\\n(.fu%2u))*.13m)
. ds #V .6m
. ds #F 0
. ds #[ \&
. ds #] \&
.\}
. \" simple accents for nroff and troff
.if n \{\
. ds ' \&
. ds ` \&
. ds ^ \&
. ds , \&
. ds ~ ~
. ds /
.\}
.if t \{\
. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
.\}
. \" troff and (daisy-wheel) nroff accents
.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
.ds 8 \h'\*(#H'\(*b\h'-\*(#H'
.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
.ds ae a\h'-(\w'a'u*4/10)'e
.ds Ae A\h'-(\w'A'u*4/10)'E
. \" corrections for vroff
.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
. \" for low resolution devices (crt and lpr)
.if \n(.H>23 .if \n(.V>19 \
\{\
. ds : e
. ds 8 ss
. ds o a
. ds d- d\h'-1'\(ga
. ds D- D\h'-1'\(hy
. ds th \o'bp'
. ds Th \o'LP'
. ds ae ae
. ds Ae AE
.\}
.rm #[ #] #H #V #F C
.\" ========================================================================
.\"
.IX Title "PERLEBCDIC 1"
.TH PERLEBCDIC 1 "2004-11-05" "perl v5.8.6" "Perl Programmers Reference Guide"
.SH "NAME"
perlebcdic \- Considerations for running Perl on EBCDIC platforms
.SH "DESCRIPTION"
.IX Header "DESCRIPTION"
An exploration of some of the issues facing Perl programmers
on \s-1EBCDIC\s0 based computers. We do not cover localization,
internationalization, or multi byte character set issues other
than some discussion of \s-1UTF\-8\s0 and \s-1UTF\-EBCDIC\s0.
.PP
Portions that are still incomplete are marked with \s-1XXX\s0.
.SH "COMMON CHARACTER CODE SETS"
.IX Header "COMMON CHARACTER CODE SETS"
.Sh "\s-1ASCII\s0"
.IX Subsection "ASCII"
The American Standard Code for Information Interchange is a set of
integers running from 0 to 127 (decimal) that imply character
interpretation by the display and other system(s) of computers.
The range 0..127 can be covered by setting the bits in a 7\-bit binary
digit, hence the set is sometimes referred to as a \*(L"7\-bit \s-1ASCII\s0\*(R".
\&\s-1ASCII\s0 was described by the American National Standards Institute
document \s-1ANSI\s0 X3.4\-1986. It was also described by \s-1ISO\s0 646:1991
(with localization for currency symbols). The full \s-1ASCII\s0 set is
given in the table below as the first 128 elements. Languages that
can be written adequately with the characters in \s-1ASCII\s0 include
English, Hawaiian, Indonesian, Swahili and some Native American
languages.
.PP
There are many character sets that extend the range of integers
from 0..2**7\-1 up to 2**8\-1, or 8 bit bytes (octets if you prefer).
One common one is the \s-1ISO\s0 8859\-1 character set.
.Sh "\s-1ISO\s0 8859"
.IX Subsection "ISO 8859"
The \s-1ISO\s0 8859\-$n are a collection of character code sets from the
International Organization for Standardization (\s-1ISO\s0) each of which
adds characters to the \s-1ASCII\s0 set that are typically found in European
languages many of which are based on the Roman, or Latin, alphabet.
.Sh "Latin 1 (\s-1ISO\s0 8859\-1)"
.IX Subsection "Latin 1 (ISO 8859-1)"
A particular 8\-bit extension to \s-1ASCII\s0 that includes grave and acute
accented Latin characters. Languages that can employ \s-1ISO\s0 8859\-1
include all the languages covered by \s-1ASCII\s0 as well as Afrikaans,
Albanian, Basque, Catalan, Danish, Faroese, Finnish, Norwegian,
Portuguese, Spanish, and Swedish. Dutch is covered albeit without
the ij ligature. French is covered too but without the oe ligature.
German can use \s-1ISO\s0 8859\-1 but must do so without German-style
quotation marks. This set is based on Western European extensions
to \s-1ASCII\s0 and is commonly encountered in world wide web work.
In \s-1IBM\s0 character code set identification terminology \s-1ISO\s0 8859\-1 is
also known as \s-1CCSID\s0 819 (or sometimes 0819 or even 00819).
.Sh "\s-1EBCDIC\s0"
.IX Subsection "EBCDIC"
The Extended Binary Coded Decimal Interchange Code refers to a
large collection of slightly different single and multi byte
coded character sets that are different from \s-1ASCII\s0 or \s-1ISO\s0 8859\-1
and typically run on host computers. The \s-1EBCDIC\s0 encodings derive
from 8 bit byte extensions of Hollerith punched card encodings.
The layout on the cards was such that high bits were set for the
upper and lower case alphabet characters [a\-z] and [A\-Z], but there
were gaps within each latin alphabet range.
.PP
Some \s-1IBM\s0 \s-1EBCDIC\s0 character sets may be known by character code set
identification numbers (\s-1CCSID\s0 numbers) or code page numbers. Leading
zero digits in \s-1CCSID\s0 numbers within this document are insignificant.
E.g. \s-1CCSID\s0 0037 may be referred to as 37 in places.
.Sh "13 variant characters"
.IX Subsection "13 variant characters"
Among \s-1IBM\s0 \s-1EBCDIC\s0 character code sets there are 13 characters that
are often mapped to different integer values. Those characters
are known as the 13 \*(L"variant\*(R" characters and are:
.PP
.Vb 1
\& \e [ ] { } ^ ~ ! # | $ @ `
.Ve
.Sh "0037"
.IX Subsection "0037"
Character code set \s-1ID\s0 0037 is a mapping of the \s-1ASCII\s0 plus Latin\-1
characters (i.e. \s-1ISO\s0 8859\-1) to an \s-1EBCDIC\s0 set. 0037 is used
in North American English locales on the \s-1OS/400\s0 operating system
that runs on \s-1AS/400\s0 computers. \s-1CCSID\s0 37 differs from \s-1ISO\s0 8859\-1
in 237 places, in other words they agree on only 19 code point values.
.Sh "1047"
.IX Subsection "1047"
Character code set \s-1ID\s0 1047 is also a mapping of the \s-1ASCII\s0 plus
Latin\-1 characters (i.e. \s-1ISO\s0 8859\-1) to an \s-1EBCDIC\s0 set. 1047 is
used under Unix System Services for \s-1OS/390\s0 or z/OS, and OpenEdition
for \s-1VM/ESA\s0. \s-1CCSID\s0 1047 differs from \s-1CCSID\s0 0037 in eight places.
.Sh "POSIX-BC"
.IX Subsection "POSIX-BC"
The \s-1EBCDIC\s0 code page in use on Siemens' \s-1BS2000\s0 system is distinct from
1047 and 0037. It is identified below as the POSIX-BC set.
.Sh "Unicode code points versus \s-1EBCDIC\s0 code points"
.IX Subsection "Unicode code points versus EBCDIC code points"
In Unicode terminology a \fIcode point\fR is the number assigned to a
character: for example, in \s-1EBCDIC\s0 the character \*(L"A\*(R" is usually assigned
the number 193. In Unicode the character \*(L"A\*(R" is assigned the number 65.
This causes a problem with the semantics of the pack/unpack \*(L"U\*(R", which
are supposed to pack Unicode code points to characters and back to numbers.
The problem is: which code points to use for code points less than 256?
(for 256 and over there's no problem: Unicode code points are used)
In \s-1EBCDIC\s0, for the low 256 the \s-1EBCDIC\s0 code points are used. This
means that the equivalences
.PP
.Vb 2
\& pack("U", ord($character)) eq $character
\& unpack("U", $character) == ord $character
.Ve
.PP
will hold. (If Unicode code points were applied consistently over
all the possible code points, pack(\*(L"U\*(R",ord(\*(L"A\*(R")) would in \s-1EBCDIC\s0
equal \fIA with acute\fR or chr(101), and unpack(\*(L"U\*(R", \*(L"A\*(R") would equal
65, or \fInon-breaking space\fR, not 193, or ord \*(L"A\*(R".)
.Sh "Remaining Perl Unicode problems in \s-1EBCDIC\s0"
.IX Subsection "Remaining Perl Unicode problems in EBCDIC"
.IP "\(bu" 4
Many of the remaining seem to be related to case-insensitive matching:
for example, \f(CW\*(C`/[\ex{131}]/\*(C'\fR (\s-1LATIN\s0 \s-1SMALL\s0 \s-1LETTER\s0 \s-1DOTLESS\s0 I) does
not match \*(L"I\*(R" case\-insensitively, as it should under Unicode.
(The match succeeds in ASCII-derived platforms.)
.IP "\(bu" 4
The extensions Unicode::Collate and Unicode::Normalized are not
supported under \s-1EBCDIC\s0, likewise for the encoding pragma.
.Sh "Unicode and \s-1UTF\s0"
.IX Subsection "Unicode and UTF"
\&\s-1UTF\s0 is a Unicode Transformation Format. \s-1UTF\-8\s0 is a Unicode conforming
representation of the Unicode standard that looks very much like \s-1ASCII\s0.
UTF-EBCDIC is an attempt to represent Unicode characters in an \s-1EBCDIC\s0
transparent manner.
.Sh "Using Encode"
.IX Subsection "Using Encode"
Starting from Perl 5.8 you can use the standard new module Encode
to translate from \s-1EBCDIC\s0 to Latin\-1 code points
.PP
.Vb 1
\& use Encode 'from_to';
.Ve
.PP
.Vb 1
\& my %ebcdic = ( 176 => 'cp37', 95 => 'cp1047', 106 => 'posix-bc' );
.Ve
.PP
.Vb 3
\& # $a is in EBCDIC code points
\& from_to($a, $ebcdic{ord '^'}, 'latin1');
\& # $a is ISO 8859-1 code points
.Ve
.PP
and from Latin\-1 code points to \s-1EBCDIC\s0 code points
.PP
.Vb 1
\& use Encode 'from_to';
.Ve
.PP
.Vb 1
\& my %ebcdic = ( 176 => 'cp37', 95 => 'cp1047', 106 => 'posix-bc' );
.Ve
.PP
.Vb 3
\& # $a is ISO 8859-1 code points
\& from_to($a, 'latin1', $ebcdic{ord '^'});
\& # $a is in EBCDIC code points
.Ve
.PP
For doing I/O it is suggested that you use the autotranslating features
of PerlIO, see perluniintro.
.PP
Since version 5.8 Perl uses the new PerlIO I/O library. This enables
you to use different encodings per \s-1IO\s0 channel. For example you may use
.PP
.Vb 9
\& use Encode;
\& open($f, ">:encoding(ascii)", "test.ascii");
\& print $f "Hello World!\en";
\& open($f, ">:encoding(cp37)", "test.ebcdic");
\& print $f "Hello World!\en";
\& open($f, ">:encoding(latin1)", "test.latin1");
\& print $f "Hello World!\en";
\& open($f, ">:encoding(utf8)", "test.utf8");
\& print $f "Hello World!\en";
.Ve
.PP
to get two files containing \*(L"Hello World!\en\*(R" in \s-1ASCII\s0, \s-1CP\s0 37 \s-1EBCDIC\s0,
\&\s-1ISO\s0 8859\-1 (Latin\-1) (in this example identical to \s-1ASCII\s0) respective
UTF-EBCDIC (in this example identical to normal \s-1EBCDIC\s0). See the
documentation of Encode::PerlIO for details.
.PP
As the PerlIO layer uses raw \s-1IO\s0 (bytes) internally, all this totally
ignores things like the type of your filesystem (\s-1ASCII\s0 or \s-1EBCDIC\s0).
.SH "SINGLE OCTET TABLES"
.IX Header "SINGLE OCTET TABLES"
The following tables list the \s-1ASCII\s0 and Latin 1 ordered sets including
the subsets: C0 controls (0..31), \s-1ASCII\s0 graphics (32..7e), delete (7f),
C1 controls (80..9f), and Latin\-1 (a.k.a. \s-1ISO\s0 8859\-1) (a0..ff). In the
table non-printing control character names as well as the Latin 1
extensions to \s-1ASCII\s0 have been labelled with character names roughly
corresponding to \fIThe Unicode Standard, Version 3.0\fR albeit with
substitutions such as s/LATIN// and s/VULGAR// in all cases,
s/CAPITAL \s-1LETTER//\s0 in some cases, and s/SMALL \s-1LETTER\s0 ([A\-Z])/\el$1/
in some other cases (the \f(CW\*(C`charnames\*(C'\fR pragma names unfortunately do
not list explicit names for the C0 or C1 control characters). The
\&\*(L"names\*(R" of the C1 control set (128..159 in \s-1ISO\s0 8859\-1) listed here are
somewhat arbitrary. The differences between the 0037 and 1047 sets are
flagged with ***. The differences between the 1047 and POSIX-BC sets
are flagged with ###. All \fIord()\fR numbers listed are decimal. If you
would rather see this table listing octal values then run the table
(that is, the pod version of this document since this recipe may not
work with a pod2_other_format translation) through:
.IP "recipe 0" 4
.IX Item "recipe 0"
.PP
.Vb 2
\& perl -ne 'if(/(.{33})(\ed+)\es+(\ed+)\es+(\ed+)\es+(\ed+)/)' \e
\& -e '{printf("%s%-9o%-9o%-9o%o\en",$1,$2,$3,$4,$5)}' perlebcdic.pod
.Ve
.PP
If you want to retain the UTF-x code points then in script form you
might want to write:
.IP "recipe 1" 4
.IX Item "recipe 1"
.PP
.Vb 14
\& open(FH,") {
\& if (/(.{33})(\ed+)\es+(\ed+)\es+(\ed+)\es+(\ed+)\es+(\ed+)\e.?(\ed*)\es+(\ed+)\e.?(\ed*)/) {
\& if ($7 ne '' && $9 ne '') {
\& printf("%s%-9o%-9o%-9o%-9o%-3o.%-5o%-3o.%o\en",$1,$2,$3,$4,$5,$6,$7,$8,$9);
\& }
\& elsif ($7 ne '') {
\& printf("%s%-9o%-9o%-9o%-9o%-3o.%-5o%o\en",$1,$2,$3,$4,$5,$6,$7,$8);
\& }
\& else {
\& printf("%s%-9o%-9o%-9o%-9o%-9o%o\en",$1,$2,$3,$4,$5,$6,$8);
\& }
\& }
\& }
.Ve
.PP
If you would rather see this table listing hexadecimal values then
run the table through:
.IP "recipe 2" 4
.IX Item "recipe 2"
.PP
.Vb 2
\& perl -ne 'if(/(.{33})(\ed+)\es+(\ed+)\es+(\ed+)\es+(\ed+)/)' \e
\& -e '{printf("%s%-9X%-9X%-9X%X\en",$1,$2,$3,$4,$5)}' perlebcdic.pod
.Ve
.PP
Or, in order to retain the UTF-x code points in hexadecimal:
.IP "recipe 3" 4
.IX Item "recipe 3"
.PP
.Vb 14
\& open(FH,") {
\& if (/(.{33})(\ed+)\es+(\ed+)\es+(\ed+)\es+(\ed+)\es+(\ed+)\e.?(\ed*)\es+(\ed+)\e.?(\ed*)/) {
\& if ($7 ne '' && $9 ne '') {
\& printf("%s%-9X%-9X%-9X%-9X%-2X.%-6X%-2X.%X\en",$1,$2,$3,$4,$5,$6,$7,$8,$9);
\& }
\& elsif ($7 ne '') {
\& printf("%s%-9X%-9X%-9X%-9X%-2X.%-6X%X\en",$1,$2,$3,$4,$5,$6,$7,$8);
\& }
\& else {
\& printf("%s%-9X%-9X%-9X%-9X%-9X%X\en",$1,$2,$3,$4,$5,$6,$8);
\& }
\& }
\& }
.Ve
.PP
.Vb 260
\& incomp- incomp-
\& 8859-1 lete lete
\& chr 0819 0037 1047 POSIX-BC UTF-8 UTF-EBCDIC
\& ------------------------------------------------------------------------------------
\& 0 0 0 0 0 0
\& 1 1 1 1 1 1
\& 2 2 2 2 2 2
\& 3 3 3 3 3 3
\& 4 55 55 55 4 55
\& 5 45 45 45 5 45
\& 6 46 46 46 6 46
\& 7 47 47 47 7 47
\& 8 22 22 22 8 22
\& 9 5 5 5 9 5
\& 10 37 21 21 10 21 ***
\& 11 11 11 11 11 11
\&
www.fiveanddime.net
