be more carful wrt. signed vs. unsigned argument passing
[paraslash.git] / crypt.c
1 /*
2 * Copyright (C) 2005-2007 Andre Noll <maan@systemlinux.org>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111, USA.
17 */
18
19 /** \file crypt.c openssl-based RSA encryption/decryption routines */
20
21 #include "para.h"
22 #include "error.h"
23 #include "string.h"
24 #include "crypt.h"
25
26 static EVP_PKEY *load_key(const char *file, int private)
27 {
28 BIO *key;
29 EVP_PKEY *pkey = NULL;
30
31 key = BIO_new(BIO_s_file());
32 if (!key)
33 return NULL;
34 if (BIO_read_filename(key, file) > 0) {
35 if (private == LOAD_PRIVATE_KEY)
36 pkey = PEM_read_bio_PrivateKey(key, NULL, NULL, NULL);
37 else
38 pkey = PEM_read_bio_PUBKEY(key, NULL, NULL, NULL);
39 }
40 BIO_free(key);
41 return pkey;
42 }
43
44 /**
45 * read an RSA key from a file
46 *
47 * \param key_file the file containing the key
48 * \param rsa RSA structure is returned here
49 * \param private if non-zero, read the private key, otherwise the public key
50 *
51 * \return The size of the RSA key on success, negative on errors.
52 *
53 * \sa openssl(1), rsa(1).
54 */
55 int get_rsa_key(char *key_file, RSA **rsa, int private)
56 {
57 EVP_PKEY *key = load_key(key_file, private);
58
59 if (!key)
60 return (private == LOAD_PRIVATE_KEY)? -E_PRIVATE_KEY : -E_PUBLIC_KEY;
61 *rsa = EVP_PKEY_get1_RSA(key);
62 EVP_PKEY_free(key);
63 if (!*rsa)
64 return -E_RSA;
65 return RSA_size(*rsa);
66 }
67
68 /**
69 * free an RSA structure
70 *
71 * \param rsa pointer to the RSA struct to free
72 *
73 * This must be called for any key obtained by get_rsa_key().
74 */
75 void rsa_free(RSA *rsa)
76 {
77 if (rsa)
78 RSA_free(rsa);
79 }
80
81 /**
82 * decrypt a buffer using an RSA key
83 *
84 * \param key_file full path of the rsa key
85 * \param outbuf the output buffer
86 * \param inbuf the encrypted input buffer
87 * \param rsa_inlen the length of \a inbuf
88 *
89 * The \a outbuf must be large enough to hold at least \a rsa_inlen bytes.
90 *
91 * \return The size of the recovered plaintext on success, negative on errors.
92 *
93 * \sa RSA_private_decrypt(3)
94 **/
95 int para_decrypt_buffer(char *key_file, unsigned char *outbuf, unsigned char *inbuf,
96 unsigned rsa_inlen)
97 {
98 RSA *rsa;
99 int ret, inlen = rsa_inlen;
100
101 if (inlen < 0)
102 return -E_RSA;
103 ret = get_rsa_key(key_file, &rsa, LOAD_PRIVATE_KEY);
104 if (ret < 0)
105 return ret;
106 ret = RSA_private_decrypt(inlen, inbuf, outbuf, rsa, RSA_PKCS1_PADDING);
107 rsa_free(rsa);
108 return (ret > 0)? ret : -E_DECRYPT;
109 }
110
111 /**
112 * decrypt the challenge number sent by para_server
113 *
114 * \param key_file full path of the rsa key
115 * \param challenge_nr result is stored here
116 * \param inbuf the input buffer
117 * \param rsa_inlen the length of \a inbuf
118 *
119 * \return positive on success, negative on errors
120 *
121 * \sa para_decrypt_buffer()
122 */
123 int para_decrypt_challenge(char *key_file, long unsigned *challenge_nr,
124 unsigned char *inbuf, unsigned rsa_inlen)
125 {
126 unsigned char *rsa_out = OPENSSL_malloc(rsa_inlen + 1);
127 int ret = para_decrypt_buffer(key_file, rsa_out, inbuf, rsa_inlen);
128
129 if (ret >= 0) {
130 rsa_out[ret] = '\0';
131 ret = sscanf((char *)rsa_out, "%lu", challenge_nr) == 1?
132 1 : -E_CHALLENGE;
133 }
134 OPENSSL_free(rsa_out);
135 return ret;
136 }
137
138 /**
139 * encrypt a buffer using an RSA key
140 *
141 * \param rsa: public rsa key
142 * \param inbuf the input buffer
143 * \param len the length of \a inbuf
144 * \param outbuf the output buffer
145 *
146 * \return The size of the encrypted data on success, negative on errors
147 *
148 * \sa RSA_public_encrypt(3)
149 */
150 int para_encrypt_buffer(RSA *rsa, unsigned char *inbuf,
151 unsigned len, unsigned char *outbuf)
152 {
153 int ret, flen = len; /* RSA_public_encrypt expects a signed int */
154
155 if (flen < 0)
156 return -E_ENCRYPT;
157 ret = RSA_public_encrypt(flen, inbuf, outbuf, rsa, RSA_PKCS1_PADDING);
158 return ret < 0? -E_ENCRYPT : ret;
159 }
160
161 /**
162 * encrypt the given challenge number
163 *
164 * \param rsa: public rsa key
165 * \param challenge_nr the number to be encrypted
166 * \param outbuf the output buffer
167 *
168 * \a outbuf must be at least 64 bytes long
169 *
170 * \return The size of the encrypted data on success, negative on errors
171 *
172 * \sa para_encrypt_buffer()
173 *
174 */
175 int para_encrypt_challenge(RSA* rsa, long unsigned challenge_nr,
176 unsigned char *outbuf)
177 {
178 unsigned char *inbuf = (unsigned char*) make_message("%lu", challenge_nr);
179 int ret = para_encrypt_buffer(rsa, inbuf, strlen((char *)inbuf), outbuf);
180 free(inbuf);
181 return ret;
182 }
183