--- linux-2.6.8.1/include/linux/pid.h.pid 2004-08-14 14:54:52.000000000 +0400 +++ linux-2.6.8.1/include/linux/pid.h 2004-08-31 15:28:07.257912920 +0400 @@ -10,55 +10,44 @@ enum pid_type PIDTYPE_MAX }; -struct pid +struct pid_struct { int nr; - atomic_t count; - struct task_struct *task; - struct list_head task_list; - struct list_head hash_chain; -}; - -struct pid_link -{ - struct list_head pid_chain; - struct pid *pidptr; - struct pid pid; + struct list_head hash_list; + struct list_head pid_list; /* list of the same pids */ }; #define pid_task(elem, type) \ - list_entry(elem, struct task_struct, pids[type].pid_chain) + list_entry(elem, struct task_struct, pids[type].pid_list) /* - * attach_pid() and link_pid() must be called with the tasklist_lock - * write-held. + * attach_pid() and detach_pid() must be called with the + * tasklist_lock write-held. */ extern int FASTCALL(attach_pid(struct task_struct *task, enum pid_type type, int nr)); -extern void FASTCALL(link_pid(struct task_struct *task, struct pid_link *link, struct pid *pid)); - -/* - * detach_pid() must be called with the tasklist_lock write-held. - */ extern void FASTCALL(detach_pid(struct task_struct *task, enum pid_type)); /* * look up a PID in the hash table. Must be called with the tasklist_lock * held. */ -extern struct pid *FASTCALL(find_pid(enum pid_type, int)); +extern struct pid_struct *FASTCALL(find_pid(enum pid_type, int)); extern int alloc_pidmap(void); extern void FASTCALL(free_pidmap(int)); extern void switch_exec_pids(struct task_struct *leader, struct task_struct *thread); -#define for_each_task_pid(who, type, task, elem, pid) \ - if ((pid = find_pid(type, who))) \ - for (elem = pid->task_list.next, \ - prefetch(elem->next), \ - task = pid_task(elem, type); \ - elem != &pid->task_list; \ - elem = elem->next, prefetch(elem->next), \ - task = pid_task(elem, type)) +#define do_each_task_pid(who, type, task) \ + if ((task = find_task_by_pid_type(type, who))) { \ + prefetch(task->pids[type].pid_list.next); \ + do { + +#define while_each_task_pid(who, type, task) \ + task = pid_task(task->pids[type].pid_list.next, \ + type); \ + prefetch(task->pids[type].pid_list.next); \ + } while (list_empty(&task->pids[type].hash_list)); \ + } #endif /* _LINUX_PID_H */ --- linux-2.6.8.1/include/linux/sched.h.pid 2004-08-14 14:54:49.000000000 +0400 +++ linux-2.6.8.1/include/linux/sched.h 2004-08-31 12:28:14.000000000 +0400 @@ -444,7 +444,7 @@ struct task_struct { struct task_struct *group_leader; /* threadgroup leader */ /* PID/PID hash table linkage. */ - struct pid_link pids[PIDTYPE_MAX]; + struct pid_struct pids[PIDTYPE_MAX]; wait_queue_head_t wait_chldexit; /* for wait4() */ struct completion *vfork_done; /* for vfork() */ @@ -727,7 +727,8 @@ extern struct task_struct init_task; extern struct mm_struct init_mm; -extern struct task_struct *find_task_by_pid(int pid); +#define find_task_by_pid(nr) find_task_by_pid_type(PIDTYPE_PID, nr) +extern struct task_struct *find_task_by_pid_type(int type, int pid); extern void set_special_pids(pid_t session, pid_t pgrp); extern void __set_special_pids(pid_t session, pid_t pgrp); @@ -930,9 +931,7 @@ extern task_t * FASTCALL(next_thread(con static inline int thread_group_empty(task_t *p) { - struct pid *pid = p->pids[PIDTYPE_TGID].pidptr; - - return pid->task_list.next->next == &pid->task_list; + return list_empty(&p->pids[PIDTYPE_TGID].pid_list); } #define delay_group_leader(p) \ --- linux-2.6.8.1/drivers/char/tty_io.c.pid 2004-08-14 14:55:34.000000000 +0400 +++ linux-2.6.8.1/drivers/char/tty_io.c 2004-08-31 12:13:14.000000000 +0400 @@ -410,7 +410,6 @@ void do_tty_hangup(void *data) struct file * cons_filp = NULL; struct file *filp, *f = NULL; struct task_struct *p; - struct pid *pid; int closecount = 0, n; if (!tty) @@ -481,8 +480,7 @@ void do_tty_hangup(void *data) read_lock(&tasklist_lock); if (tty->session > 0) { - struct list_head *l; - for_each_task_pid(tty->session, PIDTYPE_SID, p, l, pid) { + do_each_task_pid(tty->session, PIDTYPE_SID, p) { if (p->signal->tty == tty) p->signal->tty = NULL; if (!p->signal->leader) @@ -491,7 +489,7 @@ void do_tty_hangup(void *data) send_group_sig_info(SIGCONT, SEND_SIG_PRIV, p); if (tty->pgrp > 0) p->signal->tty_old_pgrp = tty->pgrp; - } + } while_each_task_pid(tty->session, PIDTYPE_SID, p); } read_unlock(&tasklist_lock); @@ -563,8 +561,6 @@ void disassociate_ctty(int on_exit) { struct tty_struct *tty; struct task_struct *p; - struct list_head *l; - struct pid *pid; int tty_pgrp = -1; lock_kernel(); @@ -593,8 +589,9 @@ void disassociate_ctty(int on_exit) tty->pgrp = -1; read_lock(&tasklist_lock); - for_each_task_pid(current->signal->session, PIDTYPE_SID, p, l, pid) + do_each_task_pid(current->signal->session, PIDTYPE_SID, p) { p->signal->tty = NULL; + } while_each_task_pid(current->signal->session, PIDTYPE_SID, p); read_unlock(&tasklist_lock); unlock_kernel(); } @@ -1235,15 +1232,15 @@ static void release_dev(struct file * fi */ if (tty_closing || o_tty_closing) { struct task_struct *p; - struct list_head *l; - struct pid *pid; read_lock(&tasklist_lock); - for_each_task_pid(tty->session, PIDTYPE_SID, p, l, pid) + do_each_task_pid(tty->session, PIDTYPE_SID, p) { p->signal->tty = NULL; + } while_each_task_pid(tty->session, PIDTYPE_SID, p); if (o_tty) - for_each_task_pid(o_tty->session, PIDTYPE_SID, p,l, pid) + do_each_task_pid(o_tty->session, PIDTYPE_SID, p) { p->signal->tty = NULL; + } while_each_task_pid(o_tty->session, PIDTYPE_SID, p); read_unlock(&tasklist_lock); } @@ -1606,8 +1603,6 @@ static int fionbio(struct file *file, in static int tiocsctty(struct tty_struct *tty, int arg) { - struct list_head *l; - struct pid *pid; task_t *p; if (current->signal->leader && @@ -1630,8 +1625,9 @@ static int tiocsctty(struct tty_struct * */ read_lock(&tasklist_lock); - for_each_task_pid(tty->session, PIDTYPE_SID, p, l, pid) + do_each_task_pid(tty->session, PIDTYPE_SID, p) { p->signal->tty = NULL; + } while_each_task_pid(tty->session, PIDTYPE_SID, p); read_unlock(&tasklist_lock); } else return -EPERM; @@ -1938,8 +1934,6 @@ static void __do_SAK(void *arg) #else struct tty_struct *tty = arg; struct task_struct *p; - struct list_head *l; - struct pid *pid; int session; int i; struct file *filp; @@ -1952,7 +1946,7 @@ static void __do_SAK(void *arg) if (tty->driver->flush_buffer) tty->driver->flush_buffer(tty); read_lock(&tasklist_lock); - for_each_task_pid(session, PIDTYPE_SID, p, l, pid) { + do_each_task_pid(session, PIDTYPE_SID, p) { if (p->signal->tty == tty || session > 0) { printk(KERN_NOTICE "SAK: killed process %d" " (%s): p->signal->session==tty->session\n", @@ -1979,7 +1973,7 @@ static void __do_SAK(void *arg) spin_unlock(&p->files->file_lock); } task_unlock(p); - } + } while_each_task_pid(session, PIDTYPE_SID, p); read_unlock(&tasklist_lock); #endif } --- linux-2.6.8.1/fs/proc/base.c.pid 2004-08-14 14:55:35.000000000 +0400 +++ linux-2.6.8.1/fs/proc/base.c 2004-08-31 13:27:15.000000000 +0400 @@ -767,10 +767,9 @@ static struct inode_operations proc_pid_ .follow_link = proc_pid_follow_link }; -static int pid_alive(struct task_struct *p) +static inline int pid_alive(struct task_struct *p) { - BUG_ON(p->pids[PIDTYPE_PID].pidptr != &p->pids[PIDTYPE_PID].pid); - return atomic_read(&p->pids[PIDTYPE_PID].pid.count); + return p->pids[PIDTYPE_PID].nr != 0; } #define NUMBUF 10 --- linux-2.6.8.1/fs/fcntl.c.pid 2004-08-14 14:55:35.000000000 +0400 +++ linux-2.6.8.1/fs/fcntl.c 2004-08-31 12:13:14.000000000 +0400 @@ -497,11 +497,9 @@ void send_sigio(struct fown_struct *fown send_sigio_to_task(p, fown, fd, band); } } else { - struct list_head *l; - struct pid *pidptr; - for_each_task_pid(-pid, PIDTYPE_PGID, p, l, pidptr) { + do_each_task_pid(-pid, PIDTYPE_PGID, p) { send_sigio_to_task(p, fown, fd, band); - } + } while_each_task_pid(-pid, PIDTYPE_PGID, p); } read_unlock(&tasklist_lock); out_unlock_fown: @@ -534,11 +532,9 @@ int send_sigurg(struct fown_struct *fown send_sigurg_to_task(p, fown); } } else { - struct list_head *l; - struct pid *pidptr; - for_each_task_pid(-pid, PIDTYPE_PGID, p, l, pidptr) { + do_each_task_pid(-pid, PIDTYPE_PGID, p) { send_sigurg_to_task(p, fown); - } + } while_each_task_pid(-pid, PIDTYPE_PGID, p); } read_unlock(&tasklist_lock); out_unlock_fown: --- linux-2.6.8.1/kernel/pid.c.pid 2004-08-14 14:55:33.000000000 +0400 +++ linux-2.6.8.1/kernel/pid.c 2004-08-31 15:34:03.234796208 +0400 @@ -146,69 +146,60 @@ failure: return -1; } -fastcall struct pid *find_pid(enum pid_type type, int nr) +fastcall struct pid_struct *find_pid(enum pid_type type, int nr) { struct list_head *elem, *bucket = &pid_hash[type][pid_hashfn(nr)]; - struct pid *pid; + struct pid_struct *pid; __list_for_each(elem, bucket) { - pid = list_entry(elem, struct pid, hash_chain); + pid = list_entry(elem, struct pid_struct, hash_list); if (pid->nr == nr) return pid; } return NULL; } -void fastcall link_pid(task_t *task, struct pid_link *link, struct pid *pid) -{ - atomic_inc(&pid->count); - list_add_tail(&link->pid_chain, &pid->task_list); - link->pidptr = pid; -} - int fastcall attach_pid(task_t *task, enum pid_type type, int nr) { - struct pid *pid = find_pid(type, nr); + struct pid_struct *pid, *task_pid; - if (pid) - atomic_inc(&pid->count); - else { - pid = &task->pids[type].pid; - pid->nr = nr; - atomic_set(&pid->count, 1); - INIT_LIST_HEAD(&pid->task_list); - pid->task = task; - get_task_struct(task); - list_add(&pid->hash_chain, &pid_hash[type][pid_hashfn(nr)]); + task_pid = &task->pids[type]; + pid = find_pid(type, nr); + if (pid == NULL) { + list_add(&task_pid->hash_list, &pid_hash[type][pid_hashfn(nr)]); + INIT_LIST_HEAD(&task_pid->pid_list); + } else { + INIT_LIST_HEAD(&task_pid->hash_list); + list_add_tail(&task_pid->pid_list, &pid->pid_list); } - list_add_tail(&task->pids[type].pid_chain, &pid->task_list); - task->pids[type].pidptr = pid; + task_pid->nr = nr; return 0; } static inline int __detach_pid(task_t *task, enum pid_type type) { - struct pid_link *link = task->pids + type; - struct pid *pid = link->pidptr; + struct pid_struct *pid, *pid_next; int nr; - list_del(&link->pid_chain); - if (!atomic_dec_and_test(&pid->count)) - return 0; - + pid = &task->pids[type]; + if (!list_empty(&pid->hash_list)) { + list_del(&pid->hash_list); + if (!list_empty(&pid->pid_list)) { + pid_next = list_entry(pid->pid_list.next, + struct pid_struct, pid_list); + /* insert next pid from pid_list to hash */ + list_add(&pid_next->hash_list, + &pid_hash[type][pid_hashfn(pid_next->nr)]); + } + } + list_del(&pid->pid_list); nr = pid->nr; - list_del(&pid->hash_chain); - put_task_struct(pid->task); + pid->nr = 0; return nr; } -static void _detach_pid(task_t *task, enum pid_type type) -{ - __detach_pid(task, type); -} - void fastcall detach_pid(task_t *task, enum pid_type type) { int nr = __detach_pid(task, type); @@ -222,16 +213,16 @@ void fastcall detach_pid(task_t *task, e free_pidmap(nr); } -task_t *find_task_by_pid(int nr) +task_t *find_task_by_pid_type(int type, int nr) { - struct pid *pid = find_pid(PIDTYPE_PID, nr); + struct pid_struct *pid = find_pid(type, nr); if (!pid) return NULL; - return pid_task(pid->task_list.next, PIDTYPE_PID); + return pid_task(&pid->pid_list, type); } -EXPORT_SYMBOL(find_task_by_pid); +EXPORT_SYMBOL(find_task_by_pid_type); /* * This function switches the PIDs if a non-leader thread calls @@ -240,13 +231,13 @@ EXPORT_SYMBOL(find_task_by_pid); */ void switch_exec_pids(task_t *leader, task_t *thread) { - _detach_pid(leader, PIDTYPE_PID); - _detach_pid(leader, PIDTYPE_TGID); - _detach_pid(leader, PIDTYPE_PGID); - _detach_pid(leader, PIDTYPE_SID); + (void)__detach_pid(leader, PIDTYPE_PID); + (void)__detach_pid(leader, PIDTYPE_TGID); + (void)__detach_pid(leader, PIDTYPE_PGID); + (void)__detach_pid(leader, PIDTYPE_SID); - _detach_pid(thread, PIDTYPE_PID); - _detach_pid(thread, PIDTYPE_TGID); + (void)__detach_pid(thread, PIDTYPE_PID); + (void)__detach_pid(thread, PIDTYPE_TGID); leader->pid = leader->tgid = thread->pid; thread->pid = thread->tgid; --- linux-2.6.8.1/kernel/exit.c.pid 2004-08-14 14:56:01.000000000 +0400 +++ linux-2.6.8.1/kernel/exit.c 2004-08-31 12:15:25.000000000 +0400 @@ -131,16 +131,15 @@ void unhash_process(struct task_struct * int session_of_pgrp(int pgrp) { struct task_struct *p; - struct list_head *l; - struct pid *pid; int sid = -1; read_lock(&tasklist_lock); - for_each_task_pid(pgrp, PIDTYPE_PGID, p, l, pid) + do_each_task_pid(pgrp, PIDTYPE_PGID, p) { if (p->signal->session > 0) { sid = p->signal->session; goto out; } + } while_each_task_pid(pgrp, PIDTYPE_PGID, p); p = find_task_by_pid(pgrp); if (p) sid = p->signal->session; @@ -161,11 +160,9 @@ out: static int will_become_orphaned_pgrp(int pgrp, task_t *ignored_task) { struct task_struct *p; - struct list_head *l; - struct pid *pid; int ret = 1; - for_each_task_pid(pgrp, PIDTYPE_PGID, p, l, pid) { + do_each_task_pid(pgrp, PIDTYPE_PGID, p) { if (p == ignored_task || p->state >= TASK_ZOMBIE || p->real_parent->pid == 1) @@ -175,7 +172,7 @@ static int will_become_orphaned_pgrp(int ret = 0; break; } - } + } while_each_task_pid(pgrp, PIDTYPE_PGID, p); return ret; /* (sighing) "Often!" */ } @@ -194,10 +191,8 @@ static inline int has_stopped_jobs(int p { int retval = 0; struct task_struct *p; - struct list_head *l; - struct pid *pid; - for_each_task_pid(pgrp, PIDTYPE_PGID, p, l, pid) { + do_each_task_pid(pgrp, PIDTYPE_PGID, p) { if (p->state != TASK_STOPPED) continue; @@ -213,7 +208,7 @@ static inline int has_stopped_jobs(int p retval = 1; break; - } + } while_each_task_pid(pgrp, PIDTYPE_PGID, p); return retval; } @@ -865,9 +860,6 @@ asmlinkage long sys_exit(int error_code) task_t fastcall *next_thread(const task_t *p) { - const struct pid_link *link = p->pids + PIDTYPE_TGID; - const struct list_head *tmp, *head = &link->pidptr->task_list; - #ifdef CONFIG_SMP if (!p->sighand) BUG(); @@ -875,11 +867,7 @@ task_t fastcall *next_thread(const task_ !rwlock_is_locked(&tasklist_lock)) BUG(); #endif - tmp = link->pid_chain.next; - if (tmp == head) - tmp = head->next; - - return pid_task(tmp, PIDTYPE_TGID); + return pid_task(p->pids[PIDTYPE_TGID].pid_list.next, PIDTYPE_TGID); } EXPORT_SYMBOL(next_thread); --- linux-2.6.8.1/kernel/sys.c.pid 2004-08-14 14:54:49.000000000 +0400 +++ linux-2.6.8.1/kernel/sys.c 2004-08-31 12:13:14.000000000 +0400 @@ -310,8 +310,6 @@ asmlinkage long sys_setpriority(int whic { struct task_struct *g, *p; struct user_struct *user; - struct pid *pid; - struct list_head *l; int error = -EINVAL; if (which > 2 || which < 0) @@ -336,8 +334,9 @@ asmlinkage long sys_setpriority(int whic case PRIO_PGRP: if (!who) who = process_group(current); - for_each_task_pid(who, PIDTYPE_PGID, p, l, pid) + do_each_task_pid(who, PIDTYPE_PGID, p) { error = set_one_prio(p, niceval, error); + } while_each_task_pid(who, PIDTYPE_PGID, p); break; case PRIO_USER: if (!who) @@ -371,8 +370,6 @@ out: asmlinkage long sys_getpriority(int which, int who) { struct task_struct *g, *p; - struct list_head *l; - struct pid *pid; struct user_struct *user; long niceval, retval = -ESRCH; @@ -394,11 +391,11 @@ asmlinkage long sys_getpriority(int whic case PRIO_PGRP: if (!who) who = process_group(current); - for_each_task_pid(who, PIDTYPE_PGID, p, l, pid) { + do_each_task_pid(who, PIDTYPE_PGID, p) { niceval = 20 - task_nice(p); if (niceval > retval) retval = niceval; - } + } while_each_task_pid(who, PIDTYPE_PGID, p); break; case PRIO_USER: if (!who) @@ -1015,12 +1012,11 @@ asmlinkage long sys_setpgid(pid_t pid, p if (pgid != pid) { struct task_struct *p; - struct pid *pid; - struct list_head *l; - for_each_task_pid(pgid, PIDTYPE_PGID, p, l, pid) + do_each_task_pid(pgid, PIDTYPE_PGID, p) { if (p->signal->session == current->signal->session) goto ok_pgid; + } while_each_task_pid(pgid, PIDTYPE_PGID, p); goto out; } @@ -1098,7 +1094,7 @@ asmlinkage long sys_getsid(pid_t pid) asmlinkage long sys_setsid(void) { - struct pid *pid; + struct pid_struct *pid; int err = -EPERM; if (!thread_group_leader(current)) --- linux-2.6.8.1/kernel/signal.c.pid 2004-08-14 14:55:19.000000000 +0400 +++ linux-2.6.8.1/kernel/signal.c 2004-08-31 12:15:17.000000000 +0400 @@ -1072,8 +1072,6 @@ int group_send_sig_info(int sig, struct int __kill_pg_info(int sig, struct siginfo *info, pid_t pgrp) { struct task_struct *p; - struct list_head *l; - struct pid *pid; int retval, success; if (pgrp <= 0) @@ -1081,11 +1079,11 @@ int __kill_pg_info(int sig, struct sigin success = 0; retval = -ESRCH; - for_each_task_pid(pgrp, PIDTYPE_PGID, p, l, pid) { + do_each_task_pid(pgrp, PIDTYPE_PGID, p) { int err = group_send_sig_info(sig, info, p); success |= !err; retval = err; - } + } while_each_task_pid(pgrp, PIDTYPE_PGID, p); return success ? 0 : retval; } @@ -1112,8 +1110,6 @@ int kill_sl_info(int sig, struct siginfo *info, pid_t sid) { int err, retval = -EINVAL; - struct pid *pid; - struct list_head *l; struct task_struct *p; if (sid <= 0) @@ -1121,13 +1117,13 @@ kill_sl_info(int sig, struct siginfo *in retval = -ESRCH; read_lock(&tasklist_lock); - for_each_task_pid(sid, PIDTYPE_SID, p, l, pid) { + do_each_task_pid(sid, PIDTYPE_SID, p) { if (!p->signal->leader) continue; err = group_send_sig_info(sig, info, p); if (retval) retval = err; - } + } while_each_task_pid(sid, PIDTYPE_SID, p); read_unlock(&tasklist_lock); out: return retval; --- linux-2.6.8.1/kernel/fork.c.pid 2004-08-14 14:54:49.000000000 +0400 +++ linux-2.6.8.1/kernel/fork.c 2004-08-31 12:13:14.000000000 +0400 @@ -1083,14 +1083,13 @@ struct task_struct *copy_process(unsigne __ptrace_link(p, current->parent); attach_pid(p, PIDTYPE_PID, p->pid); + attach_pid(p, PIDTYPE_TGID, p->tgid); if (thread_group_leader(p)) { - attach_pid(p, PIDTYPE_TGID, p->tgid); attach_pid(p, PIDTYPE_PGID, process_group(p)); attach_pid(p, PIDTYPE_SID, p->signal->session); if (p->pid) __get_cpu_var(process_counts)++; - } else - link_pid(p, p->pids + PIDTYPE_TGID, &p->group_leader->pids[PIDTYPE_TGID].pid); + } nr_threads++; write_unlock_irq(&tasklist_lock); --- linux-2.6.8.1/kernel/capability.c.pid 2004-08-14 14:55:47.000000000 +0400 +++ linux-2.6.8.1/kernel/capability.c 2004-08-31 12:13:14.000000000 +0400 @@ -89,14 +89,12 @@ static inline void cap_set_pg(int pgrp, kernel_cap_t *permitted) { task_t *g, *target; - struct list_head *l; - struct pid *pid; - for_each_task_pid(pgrp, PIDTYPE_PGID, g, l, pid) { + do_each_task_pid(pgrp, PIDTYPE_PGID, g) { target = g; while_each_thread(g, target) security_capset_set(target, effective, inheritable, permitted); - } + } while_each_task_pid(pgrp, PIDTYPE_PGID, g); } /*