Re: out-of-line x86 "put_user()" implementation

From: Linus Torvalds (torvalds_at_osdl.org)
Date: 02/09/05

  • Next message: Andrew Morton: "Re: 2.6.11-rc3: Kylix application no longer works?" 
    Date:	Tue, 8 Feb 2005 18:08:21 -0800 (PST)
To: Richard Henderson <rth@twiddle.net>

    On Tue, 8 Feb 2005, Richard Henderson wrote:
    >
    > The first 3 gets lost.

    Thanks. So here's v3 (which also removes the now stale __put_user_check()
    macro).

    Andrew - do you want to put it in -mm?

                    Linus 

    ---
# This is a BitKeeper generated diff -Nru style patch.
#
# ChangeSet
#   2005/02/08 18:05:45-08:00 torvalds@evo.osdl.org 
#   x86: make "put_user()" be out-of-line
#   
#   It's really too big to be inlined.
#   
#   Ingo tests and reports: this shrinks his kernel text size by
#   about 12kB (roughly 0.5%)
# 
# arch/i386/lib/putuser.S
#   2005/02/08 18:05:32-08:00 torvalds@evo.osdl.org +87 -0
# 
# include/asm-i386/uaccess.h
#   2005/02/08 18:05:32-08:00 torvalds@evo.osdl.org +27 -13
#   x86: make "put_user()" be out-of-line
# 
# arch/i386/lib/putuser.S
#   2005/02/08 18:05:32-08:00 torvalds@evo.osdl.org +0 -0
#   BitKeeper file /home/torvalds/v2.6/linux/arch/i386/lib/putuser.S
# 
# arch/i386/lib/Makefile
#   2005/02/08 18:05:32-08:00 torvalds@evo.osdl.org +1 -1
#   x86: make "put_user()" be out-of-line
# 
# arch/i386/kernel/i386_ksyms.c
#   2005/02/08 18:05:32-08:00 torvalds@evo.osdl.org +5 -0
#   x86: make "put_user()" be out-of-line
# 
diff -Nru a/arch/i386/kernel/i386_ksyms.c b/arch/i386/kernel/i386_ksyms.c
--- a/arch/i386/kernel/i386_ksyms.c	2005-02-08 18:06:14 -08:00
+++ b/arch/i386/kernel/i386_ksyms.c	2005-02-08 18:06:14 -08:00
@@ -97,6 +97,11 @@
 EXPORT_SYMBOL(__get_user_2);
 EXPORT_SYMBOL(__get_user_4);
 
+EXPORT_SYMBOL(__put_user_1);
+EXPORT_SYMBOL(__put_user_2);
+EXPORT_SYMBOL(__put_user_4);
+EXPORT_SYMBOL(__put_user_8);
+
 EXPORT_SYMBOL(strpbrk);
 EXPORT_SYMBOL(strstr);
 
diff -Nru a/arch/i386/lib/Makefile b/arch/i386/lib/Makefile
--- a/arch/i386/lib/Makefile	2005-02-08 18:06:14 -08:00
+++ b/arch/i386/lib/Makefile	2005-02-08 18:06:14 -08:00
@@ -3,7 +3,7 @@
 #
 
 
-lib-y = checksum.o delay.o usercopy.o getuser.o memcpy.o strstr.o \
+lib-y = checksum.o delay.o usercopy.o getuser.o putuser.o memcpy.o strstr.o \
 	bitops.o
 
 lib-$(CONFIG_X86_USE_3DNOW) += mmx.o
diff -Nru a/arch/i386/lib/putuser.S b/arch/i386/lib/putuser.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/i386/lib/putuser.S	2005-02-08 18:06:14 -08:00
@@ -0,0 +1,87 @@
+/*
+ * __put_user functions.
+ *
+ * (C) Copyright 2005 Linus Torvalds
+ *
+ * These functions have a non-standard call interface
+ * to make them more efficient, especially as they
+ * return an error value in addition to the "real"
+ * return value.
+ */
+#include <asm/thread_info.h>
+
+
+/*
+ * __put_user_X
+ *
+ * Inputs:	%eax[:%edx] contains the data
+ *		%ecx contains the address
+ *
+ * Outputs:	%eax is error code (0 or -EFAULT)
+ *
+ * These functions should not modify any other registers,
+ * as they get called from within inline assembly.
+ */
+
+#define ENTER	pushl %ebx ; GET_THREAD_INFO(%ebx)
+#define EXIT	popl %ebx ; ret
+
+.text
+.align 4
+.globl __put_user_1
+__put_user_1:
+	ENTER
+	cmpl TI_addr_limit(%ebx),%ecx
+	jae bad_put_user
+1:	movb %al,(%ecx)
+	xorl %eax,%eax
+	EXIT
+
+.align 4
+.globl __put_user_2
+__put_user_2:
+	ENTER
+	movl TI_addr_limit(%ebx),%ebx
+	subl $1,%ebx
+	cmpl %ebx,%ecx
+	jae bad_put_user
+2:	movw %ax,(%ecx)
+	xorl %eax,%eax
+	EXIT
+
+.align 4
+.globl __put_user_4
+__put_user_4:
+	ENTER
+	movl TI_addr_limit(%ebx),%ebx
+	subl $3,%ebx
+	cmpl %ebx,%ecx
+	jae bad_put_user
+3:	movl %eax,(%ecx)
+	xorl %eax,%eax
+	EXIT
+
+.align 4
+.globl __put_user_8
+__put_user_8:
+	ENTER
+	movl TI_addr_limit(%ebx),%ebx
+	subl $7,%ebx
+	cmpl %ebx,%ecx
+	jae bad_put_user
+4:	movl %eax,(%ecx)
+5:	movl %edx,4(%ecx)
+	xorl %eax,%eax
+	EXIT
+
+bad_put_user:
+	movl $-14,%eax
+	EXIT
+
+.section __ex_table,"a"
+	.long 1b,bad_put_user
+	.long 2b,bad_put_user
+	.long 3b,bad_put_user
+	.long 4b,bad_put_user
+	.long 5b,bad_put_user
+.previous
diff -Nru a/include/asm-i386/uaccess.h b/include/asm-i386/uaccess.h
--- a/include/asm-i386/uaccess.h	2005-02-08 18:06:14 -08:00
+++ b/include/asm-i386/uaccess.h	2005-02-08 18:06:14 -08:00
@@ -185,6 +185,21 @@
 
 extern void __put_user_bad(void);
 
+/*
+ * Strange magic calling convention: pointer in %ecx,
+ * value in %eax(:%edx), return value in %eax, no clobbers.
+ */
+extern void __put_user_1(void);
+extern void __put_user_2(void);
+extern void __put_user_4(void);
+extern void __put_user_8(void);
+
+#define __put_user_1(x, ptr) __asm__ __volatile__("call __put_user_1":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr))
+#define __put_user_2(x, ptr) __asm__ __volatile__("call __put_user_2":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr))
+#define __put_user_4(x, ptr) __asm__ __volatile__("call __put_user_4":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr))
+#define __put_user_8(x, ptr) __asm__ __volatile__("call __put_user_8":"=A" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr))
+#define __put_user_X(x, ptr) __asm__ __volatile__("call __put_user_X":"=a" (__ret_pu):"c" (ptr))
+
 /**
  * put_user: - Write a simple value into user space.
  * @x:   Value to copy to user space.
@@ -201,9 +216,18 @@
  *
  * Returns zero on success, or -EFAULT on error.
  */
-#define put_user(x,ptr)							\
-  __put_user_check((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
-
+#define put_user(x,ptr)						\
+({	int __ret_pu;						\
+	__chk_user_ptr(ptr);					\
+	switch(sizeof(*(ptr))) {				\
+	case 1: __put_user_1(x, ptr); break;			\
+	case 2: __put_user_2(x, ptr); break;			\
+	case 4: __put_user_4(x, ptr); break;			\
+	case 8: __put_user_8(x, ptr); break;			\
+	default:__put_user_X(x, ptr); break;			\
+	}							\
+	__ret_pu;						\
+})
 
 /**
  * __get_user: - Get a simple variable from user space, with less checking.
@@ -258,16 +282,6 @@
 	__pu_err;						\
 })
 
-
-#define __put_user_check(x,ptr,size)					\
-({									\
-	long __pu_err = -EFAULT;					\
-	__typeof__(*(ptr)) __user *__pu_addr = (ptr);			\
-	might_sleep();						\
-	if (access_ok(VERIFY_WRITE,__pu_addr,size))			\
-		__put_user_size((x),__pu_addr,(size),__pu_err,-EFAULT);	\
-	__pu_err;							\
-})							
 
 #define __put_user_u64(x, addr, err)				\
 	__asm__ __volatile__(					\
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/
  • Next message: Andrew Morton: "Re: 2.6.11-rc3: Kylix application no longer works?"

    Relevant Pages

    • Re: [patch 1/1] export cpu_online_map
      ... Andrew wrote: ... macro should do so as well. ... I just sent a patch. ... send the line "unsubscribe linux-kernel" in ...
      (Linux-Kernel)
    • [GIT PULL] Userlib integration patches
      ... introduce __ASM_REG macro ... * Test whether a block of memory is a valid user space address. ... * Context: User context only. ... * Checks if a pointer to a block of memory in user space is valid. ...
      (Linux-Kernel)
    • [PATCH 0/39] Merge files at x86/lib
      ... * Test whether a block of memory is a valid user space address. ... * Context: User context only. ... * Checks if a pointer to a block of memory in user space is valid. ... * This macro copies a single simple variable from user space to kernel ...
      (Linux-Kernel)
    • Re: [RFC] SPI core -- revisited
      ... >DMA to copy data from user to kernel space? ... >cause page faults which ensure that the user space pages are paged in ... where dev is actually an SPI device. ... send the line "unsubscribe linux-kernel" in ...
      (Linux-Kernel)
    • Re: [RFC] add kobject to struct module
      ... > - Are we opening the floodgates for another round of races and driver ... If you set the "perm" portion of the module_parammacro to 0, ... perm sets the visibility in driverfs: ... send the line "unsubscribe linux-kernel" in ...
      (Linux-Kernel)